mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-05 10:26:56 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 91f2d53500 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a3e166a140f3150218b4a857fedb8749940fad0c8b053d0e3c1b957321bf5beb
|
||||
sha256-linux=dce4a8eb634a27b4589bcf6d2ebf248ede97aa59701acc8696391d2812a3177d
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
|
||||
+11
-5
@@ -417,6 +417,12 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-heal) & (binary(mrf_partial_write_test) | (binary(mrf_pipeline_test) & test(/^journal_replay_survives_/)) | test(=admin_selector_survives_sigkill_with_original_scope_and_receipts))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Bound the whole recovery fixture even if a synchronous wait prevents its
|
||||
# per-case async deadline from firing.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-heal) & binary(heal_integration_test) & test(=serial_tests::test_admin_bucket_directory_recovery_from_task_entry)'
|
||||
slow-timeout = { period = "60s", terminate-after = 5, grace-period = "10s" }
|
||||
|
||||
# Native scanner backlog fixtures include 24-set and pool-expansion cases.
|
||||
# Reserve capacity across test processes while preserving each scenario's fixed
|
||||
# budget, assertions, and internal concurrency.
|
||||
@@ -701,7 +707,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
@@ -827,7 +833,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -970,10 +976,10 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
|
||||
| File | Description | Usage |
|
||||
| :--- | :--- | :--- |
|
||||
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
|
||||
|
||||
### Docker Compose Configurations
|
||||
|
||||
|
||||
@@ -32,8 +32,6 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -769,10 +769,12 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -847,11 +849,12 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual",
|
||||
"release"
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -497,63 +497,6 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
@@ -1454,7 +1397,6 @@ jobs:
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1482,7 +1424,6 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,6 +33,32 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,19 +12,30 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -33,7 +44,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -87,206 +87,18 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Packaging tools are off on this lane, so setup does not install unzip.
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq unzip
|
||||
fi
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "unzip is required to extract console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p ./rustfs/static
|
||||
|
||||
verify_sha256() {
|
||||
local expected="$1"
|
||||
local file="$2"
|
||||
local actual=""
|
||||
local actual_line=""
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual_line=$(sha256sum -- "$file") || return 1
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual_line=$(shasum -a 256 -- "$file") || return 1
|
||||
else
|
||||
echo "No SHA-256 verification tool found" >&2
|
||||
return 1
|
||||
fi
|
||||
actual="${actual_line%%[[:space:]]*}"
|
||||
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
echo "SHA256 verified OK: $actual"
|
||||
return 0
|
||||
else
|
||||
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
local curl_auth_args=()
|
||||
console_tag=""
|
||||
|
||||
chmod +w ./rustfs/static/LICENSE || true
|
||||
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin="python"
|
||||
fi
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
echo "No Python interpreter found for release metadata parsing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
||||
fi
|
||||
|
||||
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
||||
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
|
||||
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("no console zip asset with sha256 digest found")
|
||||
PY
|
||||
) || return 1
|
||||
|
||||
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
||||
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Console release: ${console_tag}"
|
||||
echo "Downloading console asset: ${console_url}"
|
||||
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
verify_sha256 "$console_sha256" console.zip || return 2
|
||||
unzip -o console.zip -d ./rustfs/static || return 2
|
||||
}
|
||||
|
||||
if download_console_assets; then
|
||||
rm -f console.zip console-release.json
|
||||
else
|
||||
status=$?
|
||||
rm -f console.zip console-release.json
|
||||
if [[ "$status" -eq 2 ]]; then
|
||||
echo "Console asset integrity verification failed" >&2
|
||||
fi
|
||||
echo "Failed to download verified console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s ./rustfs/static/index.html ]]; then
|
||||
echo "Console asset archive is missing static/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
||||
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
||||
|
||||
- name: Build RustFS
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
shell: bash
|
||||
run: |
|
||||
# Force rebuild by touching build.rs so a restored cache cannot
|
||||
# reuse a rustfs artifact compiled before these console assets existed.
|
||||
touch rustfs/build.rs
|
||||
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Verify console
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
data_dir="$(mktemp -d)"
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$data_dir"
|
||||
' EXIT
|
||||
|
||||
if [[ ! -x "$binary" ]]; then
|
||||
echo "rustfs binary not found: ${binary}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_port="$(find_free_port)"
|
||||
console_port="$(find_free_port)"
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$binary" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
response=""
|
||||
for _ in {1..40}; do
|
||||
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "Console endpoint did not return 200 text/html: $response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
@@ -534,12 +346,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -548,90 +360,6 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
@@ -92,17 +92,6 @@ jobs:
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
@@ -110,15 +99,15 @@ jobs:
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = candidate["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
@@ -132,26 +121,6 @@ jobs:
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
@@ -255,8 +224,7 @@ jobs:
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
@@ -275,7 +243,6 @@ jobs:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
@@ -384,10 +351,7 @@ jobs:
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
|
||||
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+195
-399
File diff suppressed because it is too large
Load Diff
+17
-15
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -191,7 +191,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
@@ -213,13 +212,12 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.19.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -236,7 +234,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.39" }
|
||||
jiff = { version = "0.2.38" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -255,9 +253,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -339,7 +337,7 @@ strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
test-case = "3.4.0"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -358,13 +356,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.3.0"
|
||||
zip = "9.1.3"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -379,16 +377,16 @@ libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.1" }
|
||||
russh = { version = "0.65.0" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
hotpath = { version = "0.30.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -429,3 +427,7 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
+3
-11
@@ -16,7 +16,6 @@ FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
WORKDIR /build
|
||||
@@ -70,9 +69,6 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
@@ -99,18 +95,14 @@ RUN apk upgrade --no-cache && \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
@@ -132,6 +124,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
@@ -12,10 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.99.1-trixie
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RC_RELEASE=latest
|
||||
FROM rust:1.97.1-trixie
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
@@ -24,7 +21,6 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -40,7 +36,6 @@ COPY . .
|
||||
RUN ./scripts/static.sh
|
||||
RUN cargo run --bin gproto
|
||||
RUN cargo build --release --locked --bin rustfs
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
RUN set -eux; \
|
||||
groupadd -g 10001 rustfs; \
|
||||
@@ -55,12 +50,8 @@ RUN set -eux; \
|
||||
chown -R rustfs:rustfs /data /logs /app; \
|
||||
chmod 0750 /data /logs
|
||||
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
|
||||
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
|
||||
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
@@ -72,5 +63,5 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
+3
-11
@@ -16,7 +16,6 @@ FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
@@ -77,9 +76,6 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -103,18 +99,14 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
|
||||
RUN groupadd -g 10001 rustfs && \
|
||||
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
|
||||
@@ -139,6 +131,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
+7
-19
@@ -28,19 +28,17 @@ ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.99.1-trixie AS builder
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES
|
||||
ARG RC_RELEASE
|
||||
|
||||
# Debug: print platforms
|
||||
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
|
||||
@@ -54,7 +52,6 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -169,8 +166,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
# -----------------------------
|
||||
@@ -204,13 +199,10 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
# We rely on cargo run to build and run
|
||||
EXPOSE 9000 9001
|
||||
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
|
||||
# -----------------------------
|
||||
@@ -235,7 +227,6 @@ RUN set -eux; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
coreutils; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -257,11 +248,8 @@ RUN set -eux; \
|
||||
|
||||
# Copy the freshly built binary and the entrypoint
|
||||
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
|
||||
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
|
||||
# Default environment (override in docker run/compose as needed)
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
@@ -275,5 +263,5 @@ EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Keep root here; entrypoint will drop privileges using chroot --userspec
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
# RustFS image bootstrap
|
||||
|
||||
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
|
||||
|
||||
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
|
||||
|
||||
Optional per-bucket files use the bucket name as the filename without an extension:
|
||||
|
||||
| Directory | File format | Applied operation |
|
||||
| --- | --- | --- |
|
||||
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
|
||||
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
|
||||
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
|
||||
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
|
||||
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
|
||||
|
||||
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
|
||||
|
||||
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
|
||||
|
||||
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
|
||||
@@ -1 +0,0 @@
|
||||
[]
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"block_public_acls": true,
|
||||
"ignore_public_acls": true,
|
||||
"block_public_policy": true,
|
||||
"restrict_public_buckets": true
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
|
||||
/// Default validation mode is "hop_by_hop".
|
||||
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
|
||||
|
||||
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
|
||||
/// Environment variable to enable RFC 7239 "Forwarded" header support.
|
||||
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
|
||||
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
|
||||
/// RFC 7239 support is enabled by default.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
|
||||
|
||||
/// Environment variable for the maximum allowed proxy hops.
|
||||
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
|
||||
|
||||
@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
|
||||
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
|
||||
|
||||
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
|
||||
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -175,17 +175,12 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
|
||||
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
|
||||
|
||||
/// Environment variable for the S3 request-body inter-chunk read timeout
|
||||
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
|
||||
/// (seconds). Default: 300. Set to 0 to disable.
|
||||
///
|
||||
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
|
||||
/// while more bytes are still expected. It resets on every chunk, so it does not
|
||||
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
|
||||
/// partial body and then goes silent *without* closing the connection (no EOF).
|
||||
/// Independently of this knob, `PutObject` and `UploadPart` require at least
|
||||
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
|
||||
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
|
||||
/// that would otherwise pin upload admission indefinitely, and it stays in
|
||||
/// force when this timeout is set to 0.
|
||||
/// A reverse proxy or CDN that forwards a truncated body this way would
|
||||
/// otherwise make RustFS wait forever for bytes that never arrive; with this
|
||||
/// timeout the stalled read is aborted and logged with the received/expected
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -502,163 +502,10 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,249 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -283,9 +283,6 @@ mod ssec_copy_test;
|
||||
#[cfg(test)]
|
||||
mod multipart_storage_class_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod multipart_upload_id_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_class_capability_test;
|
||||
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart upload ID regression tests.
|
||||
//!
|
||||
//! Every operation that addresses an existing multipart upload must report an
|
||||
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
|
||||
//! the ID is malformed or is a well-formed ID of an aborted upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "multipart-upload-id";
|
||||
const KEY: &str = "target.bin";
|
||||
const COPY_SOURCE_KEY: &str = "copy-source.bin";
|
||||
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
|
||||
// the object layer rejects this ID as malformed instead of looking it up.
|
||||
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
|
||||
const PART_NUMBER: i32 = 1;
|
||||
const PART_BODY: &[u8] = b"multipart upload id regression part";
|
||||
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
|
||||
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
|
||||
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
|
||||
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
|
||||
|
||||
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
|
||||
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
|
||||
where
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let Err(error) = result else {
|
||||
return Some(format!("{operation}: succeeded"));
|
||||
};
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
|
||||
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
|
||||
}
|
||||
|
||||
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
env.create_test_bucket(BUCKET).await?;
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(COPY_SOURCE_KEY)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
|
||||
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
|
||||
let upload_part = client
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await;
|
||||
let upload_part_copy = client
|
||||
.upload_part_copy()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
|
||||
.send()
|
||||
.await;
|
||||
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
let abort = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let mismatches: Vec<String> = [
|
||||
no_such_upload_mismatch("UploadPart", upload_part),
|
||||
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
|
||||
no_such_upload_mismatch("ListParts", list_parts),
|
||||
no_such_upload_mismatch("CompleteMultipartUpload", complete),
|
||||
no_such_upload_mismatch("AbortMultipartUpload", abort),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(
|
||||
mismatches.is_empty(),
|
||||
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
|
||||
mismatches.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
|
||||
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
|
||||
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
|
||||
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
|
||||
let status = head
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|error| error.raw_response())
|
||||
.map(|response| response.status().as_u16());
|
||||
assert_eq!(
|
||||
status,
|
||||
Some(OBJECT_NOT_FOUND_STATUS),
|
||||
"rejected operations must not publish {KEY}: {head:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
}
|
||||
@@ -576,6 +576,41 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata_short(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_pool_meta_cas(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -94,12 +94,3 @@ pub(crate) mod scanner_peer_bucket {
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod bucket_tag_fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
|
||||
}
|
||||
|
||||
@@ -360,26 +360,12 @@ impl OpaMock {
|
||||
|
||||
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, action, bucket, department);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
|
||||
assert_eq!(payload["input"]["resource"]["object"], object);
|
||||
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
|
||||
let input = &payload["input"];
|
||||
assert_eq!(input["identity"]["account"], "opabuckettags");
|
||||
assert_eq!(input["identity"]["is_owner"], false);
|
||||
assert_eq!(input["action"], action);
|
||||
assert_eq!(input["resource"]["bucket"], bucket);
|
||||
let expected = department.map(|value| serde_json::json!([value]));
|
||||
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
@@ -692,61 +678,22 @@ async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop and reap the directory's owning server before opening the fixture store.
|
||||
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
use crate::storage_api::bucket_tag_fixture::{
|
||||
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
|
||||
};
|
||||
|
||||
let server = env
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("missing server process before offline fixture write")?;
|
||||
server.kill()?;
|
||||
server.wait()?;
|
||||
env.process = None;
|
||||
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
|
||||
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
|
||||
let context = std::sync::Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
|
||||
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
|
||||
let _cancel = context
|
||||
.background_cancel_token()
|
||||
.expect("store binds its shutdown token")
|
||||
.drop_guard();
|
||||
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let policy = metadata.policy_config_json.clone();
|
||||
assert!(!incarnation.is_nil());
|
||||
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
|
||||
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
|
||||
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
|
||||
metadata.tagging_config_xml = xml.to_vec();
|
||||
metadata.save_with_store(store.clone()).await?;
|
||||
let persisted = load_bucket_metadata(store, bucket).await?;
|
||||
assert_eq!(persisted.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(persisted.policy_config_json, policy);
|
||||
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
};
|
||||
|
||||
init_logging();
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let opa_url = opa.url.clone();
|
||||
let server_env = [
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
])
|
||||
.await?;
|
||||
let secret = uuid::Uuid::new_v4().to_string();
|
||||
create_user(&env, "opabuckettags", &secret).await?;
|
||||
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
|
||||
@@ -786,7 +733,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
let head = client
|
||||
client
|
||||
.head_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
@@ -794,8 +741,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
assert_eq!(head.tag_count(), Some(1));
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
client.list_objects_v2().bucket(source).send().await?;
|
||||
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
|
||||
client.list_object_versions().bucket(source).send().await?;
|
||||
@@ -805,65 +750,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
admin.create_bucket().bucket(destination).send().await?;
|
||||
set_department(&admin, destination, "engineering").await?;
|
||||
let copy_source = format!("{source}/report?versionId={version}");
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("denied COPY and REPLACE must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
set_department(&admin, destination, "finance").await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
// The access hook checks preliminary tags, then the copy use case checks
|
||||
// the exact source reader's tags before publishing the destination.
|
||||
for _ in 0..2 {
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
}
|
||||
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
|
||||
assert_eq!(tags.tag_set().len(), 1);
|
||||
assert_eq!(tags.tag_set()[0].key(), "department");
|
||||
assert_eq!(
|
||||
tags.tag_set()[0].value(),
|
||||
if directive == TaggingDirective::Copy {
|
||||
"engineering"
|
||||
} else {
|
||||
"legal"
|
||||
}
|
||||
);
|
||||
}
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
@@ -939,14 +847,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("denied-source")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied source must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
set_department(&client, source, "finance")
|
||||
.await
|
||||
.expect_err("requested tags must not authorize their own mutation");
|
||||
@@ -985,7 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let invalid_write = admin
|
||||
admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
.tagging(
|
||||
@@ -995,25 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
|
||||
let read = client
|
||||
.get_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(destination)
|
||||
|
||||
@@ -31,7 +31,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -192,7 +192,6 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -332,8 +331,8 @@ pub mod bucket {
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -452,15 +451,9 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
|
||||
pub use crate::disk::multipart_index::{
|
||||
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
|
||||
MultipartIndexRpcRequest,
|
||||
};
|
||||
pub use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
||||
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
|
||||
@@ -473,9 +466,7 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
|
||||
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
|
||||
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
@@ -626,7 +617,7 @@ pub mod rebalance {
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
boxed_reader, compression_metadata_value, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -663,9 +654,7 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
|
||||
@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
|
||||
UploadResponseDeadline, build_remote_s3_client,
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
|
||||
// Replication has no retry budget of its own on the request path,
|
||||
// so it keeps the SDK's standard three attempts; stating it here
|
||||
// pins the behaviour to this line instead of an SDK default.
|
||||
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication request budgets, snapshotted when a target client is built.
|
||||
/// Streamed GET bodies are not subject to the upload-response budget.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TargetRequestTimeouts {
|
||||
request: Duration,
|
||||
complete: Duration,
|
||||
minimum_transfer_rate: u64,
|
||||
}
|
||||
|
||||
impl Default for TargetRequestTimeouts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request: Duration::from_secs(60),
|
||||
complete: Duration::from_secs(15 * 60),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetRequestTimeouts {
|
||||
fn from_env() -> Self {
|
||||
let defaults = Self::default();
|
||||
let seconds = |name: &str, fallback: Duration| {
|
||||
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
|
||||
Duration::from_secs(if (1..=86400).contains(&value) {
|
||||
value
|
||||
} else {
|
||||
fallback.as_secs()
|
||||
})
|
||||
};
|
||||
let minimum_transfer_rate =
|
||||
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
|
||||
Self {
|
||||
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
|
||||
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
|
||||
minimum_transfer_rate: if minimum_transfer_rate == 0 {
|
||||
defaults.minimum_transfer_rate
|
||||
} else {
|
||||
minimum_transfer_rate
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
|
||||
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
|
||||
// The SDK can make three attempts. Leave transmission headroom for
|
||||
// all three; the separate response deadline still bounds a trickling
|
||||
// target after each upload finishes.
|
||||
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
|
||||
RemoteS3RetryPolicy::Disabled => 1,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
|
||||
};
|
||||
let seconds = size
|
||||
.div_ceil(self.minimum_transfer_rate)
|
||||
.checked_mul(attempts)
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
let timeout = self
|
||||
.request
|
||||
.checked_add(Duration::from_secs(seconds))
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
if Instant::now().checked_add(timeout).is_none() {
|
||||
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
|
||||
}
|
||||
Ok(timeout)
|
||||
}
|
||||
|
||||
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
|
||||
Ok(aws_sdk_s3::config::Builder::new()
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.upload_timeout(size)?)
|
||||
.build(),
|
||||
)
|
||||
.interceptor(UploadResponseDeadline(self.request)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TargetClient {
|
||||
pub endpoint: String,
|
||||
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
|
||||
pub health_check_duration: Duration,
|
||||
pub replicate_sync: bool,
|
||||
pub client: Arc<S3Client>,
|
||||
pub timeouts: TargetRequestTimeouts,
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
@@ -2441,11 +2360,12 @@ impl TargetClient {
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
let mut config = self.timeouts.upload_override(size)?;
|
||||
if forwards_source_checksum {
|
||||
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
}
|
||||
operation = operation.config_override(config);
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
@@ -2581,7 +2501,6 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(self.timeouts.upload_override(size)?)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2619,13 +2538,6 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(
|
||||
aws_sdk_s3::config::Builder::new().timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.timeouts.complete)
|
||||
.build(),
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -3054,241 +2966,11 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TimedResponseConnector {
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: bytes::Bytes,
|
||||
frames: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ResponseDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TimedResponseConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let settings = self.clone();
|
||||
HttpConnectorFuture::new(async move {
|
||||
http_body_util::BodyExt::collect(request.into_body())
|
||||
.await
|
||||
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
|
||||
tokio::time::sleep(settings.upload_delay).await;
|
||||
let guard = ResponseDrop(Arc::clone(&settings.dropped));
|
||||
let stream = futures::stream::unfold(
|
||||
(settings.prefix_bytes, Some(settings.response), true, guard),
|
||||
move |(remaining, mut body, first, guard)| {
|
||||
let frames = Arc::clone(&settings.frames);
|
||||
async move {
|
||||
if remaining == 0 && body.is_none() {
|
||||
return None;
|
||||
}
|
||||
if !first {
|
||||
tokio::time::sleep(settings.response_delay).await;
|
||||
}
|
||||
let bytes = if remaining > 0 {
|
||||
bytes::Bytes::from_static(b" ")
|
||||
} else {
|
||||
body.take()?
|
||||
};
|
||||
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Some((
|
||||
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
|
||||
(remaining.saturating_sub(1), body, false, guard),
|
||||
))
|
||||
}
|
||||
},
|
||||
)
|
||||
.fuse();
|
||||
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
|
||||
let mut response =
|
||||
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
|
||||
response.headers_mut().insert("etag", "\"test-etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_response_target(
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: &'static [u8],
|
||||
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(TimedResponseConnector {
|
||||
upload_delay,
|
||||
response_delay,
|
||||
prefix_bytes,
|
||||
response: bytes::Bytes::from_static(response),
|
||||
frames: Arc::clone(&frames),
|
||||
dropped: Arc::clone(&dropped),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut target, _) = header_recording_target_client(Vec::new());
|
||||
target.timeouts = TargetRequestTimeouts {
|
||||
request: Duration::from_secs(2),
|
||||
complete: Duration::from_secs(10),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
};
|
||||
target.client = Arc::new(S3Client::from_conf(
|
||||
target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.http_client(http_client)
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(target.timeouts.request)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
));
|
||||
(target, frames, dropped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
|
||||
let policy = TargetRequestTimeouts::default();
|
||||
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
|
||||
assert_eq!(
|
||||
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
|
||||
Duration::from_secs(15420)
|
||||
);
|
||||
assert!(policy.upload_timeout(-1).is_err());
|
||||
let slow = TargetRequestTimeouts {
|
||||
minimum_transfer_rate: 1,
|
||||
..policy
|
||||
};
|
||||
assert!(slow.upload_timeout(i64::MAX).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
|
||||
for operation in ["put", "forwarded-checksum", "part"] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = if operation == "part" {
|
||||
target
|
||||
.put_object_part(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
1,
|
||||
size,
|
||||
ByteStream::from(body),
|
||||
&PutObjectPartOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
} else {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
if operation == "forwarded-checksum" {
|
||||
opts.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_owned(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
|
||||
);
|
||||
}
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_secs(6),
|
||||
"upload gets its full independent window: {error}"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
|
||||
"response must end before the larger total deadline"
|
||||
);
|
||||
assert!(
|
||||
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
|
||||
"trickle delivered progress before failure"
|
||||
);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
|
||||
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("slow upload, prompt response");
|
||||
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
|
||||
let response = target
|
||||
.get_object("bucket", "key", None, None, None, HeaderMap::new())
|
||||
.await
|
||||
.expect("GET headers");
|
||||
let started = tokio::time::Instant::now();
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("streamed GET remains outside upload deadline")
|
||||
.into_bytes();
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert_eq!(&body[..], b" payload");
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
|
||||
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
|
||||
for prefix in [5, 100] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = target
|
||||
.complete_multipart_upload(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if prefix == 5 {
|
||||
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(11));
|
||||
}
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_control_operation_deadline_covers_response_trickle() {
|
||||
let (target, frames, dropped) =
|
||||
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
|
||||
let started = tokio::time::Instant::now();
|
||||
assert!(target.get_bucket_versioning("bucket").await.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(3));
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
@@ -3597,7 +3279,6 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
@@ -3910,7 +3591,6 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -5487,14 +5487,6 @@ pub async fn put_restore_opts(
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = oi.expires {
|
||||
meta.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(std::io::Error::other)?,
|
||||
);
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
@@ -5514,6 +5506,7 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -7151,25 +7144,6 @@ mod tests {
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
let unread = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("open an unread transitioned response");
|
||||
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
|
||||
drop(unread);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"canceling an unread response must release the tier lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,38 +256,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -1022,20 +990,6 @@ impl BucketMetadata {
|
||||
self.encryption_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
if !data.is_empty() {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
@@ -1830,31 +1784,6 @@ mod test {
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
|
||||
let mut metadata = BucketMetadata::new("validated-tags");
|
||||
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
|
||||
.expect("empty value is valid");
|
||||
let updated = metadata.tagging_config_updated_at;
|
||||
for invalid in [
|
||||
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
|
||||
] {
|
||||
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
|
||||
assert_eq!(metadata.tagging_config_xml, valid);
|
||||
assert_eq!(metadata.tagging_config_updated_at, updated);
|
||||
}
|
||||
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("deletion repairs legacy corruption");
|
||||
assert!(metadata.tagging_config_xml.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
|
||||
@@ -3428,7 +3428,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
@@ -4556,13 +4556,8 @@ mod tests {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
|
||||
// Model legacy persisted corruption without passing the write-time validator.
|
||||
bm.tagging_config_xml = raw.to_vec();
|
||||
} else {
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
}
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
@@ -5743,8 +5738,7 @@ mod tests {
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging =
|
||||
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
@@ -5817,7 +5811,7 @@ mod tests {
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
|
||||
@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
// Exercise the real storage entry point before testing the new
|
||||
// reporting semantics: the same ledger must admit and refuse
|
||||
// these writes even when a scanner floor claims a million bytes.
|
||||
if protocol == 1 {
|
||||
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
),
|
||||
"a missing authoritative ledger must not admit a write from scanner reporting"
|
||||
);
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative usage through the production worker");
|
||||
}
|
||||
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::disk::{DiskAPI, error::DiskError};
|
||||
use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
|
||||
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
@@ -36,8 +36,6 @@ use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
|
||||
const RECONCILE_RETRY_SECONDS: i64 = 30;
|
||||
const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
|
||||
const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
@@ -98,10 +96,6 @@ struct QuotaLedger {
|
||||
version: u8,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
/// Changes on every durable write, including zero-growth reservations.
|
||||
/// Missing on an older writer's output, which invalidates a concurrent scan.
|
||||
#[serde(default)]
|
||||
writer_revision: Uuid,
|
||||
accounted_usage: u64,
|
||||
reservations: BTreeMap<Uuid, PersistedReservation>,
|
||||
#[serde(default)]
|
||||
@@ -116,7 +110,6 @@ impl QuotaLedger {
|
||||
version: QUOTA_LEDGER_FORMAT_VERSION,
|
||||
bucket_incarnation,
|
||||
quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
|
||||
writer_revision: Uuid::nil(),
|
||||
accounted_usage,
|
||||
reservations: BTreeMap::new(),
|
||||
reconcile_required: false,
|
||||
@@ -492,23 +485,22 @@ impl QuotaContext {
|
||||
let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
|
||||
let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
|
||||
let mut ledger =
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
|
||||
.await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
|
||||
let Some(mut ledger) = ledger.take() else {
|
||||
// Bootstrap has no authoritative usage yet. Release every
|
||||
// request-owned fence by returning; the worker owns no request.
|
||||
job.schedule();
|
||||
return Err(StorageError::SlowDown);
|
||||
};
|
||||
let mut ledger = load_current_ledger_locked(
|
||||
Arc::clone(&store),
|
||||
&ledger_data.bucket,
|
||||
&ledger_data.ledger_object,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let growth = reservation.growth();
|
||||
let current_usage = ledger.admitted_usage()?;
|
||||
let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if ledger.reconcile_required
|
||||
|| (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
|
||||
{
|
||||
job.schedule();
|
||||
let mut current_usage = ledger.admitted_usage()?;
|
||||
let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
|
||||
reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
|
||||
current_usage = ledger.admitted_usage()?;
|
||||
expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
}
|
||||
if growth > 0 && expected_usage > quota_limit {
|
||||
return Err(StorageError::QuotaExceeded {
|
||||
@@ -526,7 +518,7 @@ impl QuotaContext {
|
||||
});
|
||||
}
|
||||
ledger.reserve(operation_id, reservation)?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
|
||||
|
||||
Ok(QuotaReservation {
|
||||
ledger: Some(ReservationLedgerData::Single(ledger_data)),
|
||||
@@ -1090,7 +1082,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
|
||||
fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
|
||||
let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
|
||||
ledger.mark_commit_started(data.operation_id, &data.reservation)?;
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
|
||||
@@ -1121,7 +1113,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
} else {
|
||||
ledger.abort(operation_id, &reservation)?;
|
||||
}
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
|
||||
@@ -1129,15 +1121,22 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
|
||||
async fn load_current_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
ledger_object: &str,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
) -> Result<Option<QuotaLedger>> {
|
||||
match load_ledger_locked(store, ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
|
||||
Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
|
||||
) -> Result<QuotaLedger> {
|
||||
match load_ledger_locked(Arc::clone(&store), ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
|
||||
Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
Ok(_) => Err(StorageError::PartMissingOrCorrupt),
|
||||
Err(StorageError::ConfigNotFound) => Ok(None),
|
||||
Err(StorageError::ConfigNotFound) => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -1246,11 +1245,15 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
|
||||
for (object, target) in orphan_commit_targets {
|
||||
fence_namespace_mutations(&store, bucket, &object, target).await?;
|
||||
}
|
||||
let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
if ledger.reservations.is_empty() && ledger.reconcile_required {
|
||||
reconcile_exact(&store, bucket, &mut ledger).await?;
|
||||
removed = true;
|
||||
}
|
||||
if !removed && !cursor_changed {
|
||||
return Ok(());
|
||||
}
|
||||
save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
|
||||
save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
|
||||
}
|
||||
|
||||
fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
|
||||
@@ -1270,217 +1273,13 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ReconcileControl {
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
next_attempt_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ReconcileJob {
|
||||
store: Arc<ECStore>,
|
||||
bucket: String,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
type ReconcileKey = (Uuid, String, Uuid, i128);
|
||||
|
||||
fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
|
||||
static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
|
||||
ACTIVE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
struct ActiveReconciliation(ReconcileKey);
|
||||
|
||||
impl Drop for ActiveReconciliation {
|
||||
fn drop(&mut self) {
|
||||
active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconcileJob {
|
||||
fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
|
||||
Self {
|
||||
store,
|
||||
bucket,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule(self) {
|
||||
let key = (
|
||||
self.store.id,
|
||||
self.bucket.clone(),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
if !active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(key.clone())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let owner = ActiveReconciliation(key);
|
||||
// The owner carries no object, metadata, lifecycle or reservation guard.
|
||||
// Cancellation of the rejected request cannot cancel its reconciliation.
|
||||
tokio::spawn(async move {
|
||||
let _owner = owner;
|
||||
if let Err(error) = self.run().await {
|
||||
warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
|
||||
error = %error, "Quota reconciliation deferred");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn control_object(&self) -> String {
|
||||
format!(
|
||||
"{}.reconcile/{}/{}",
|
||||
ledger_object(&self.bucket),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos()
|
||||
)
|
||||
}
|
||||
|
||||
async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
|
||||
}
|
||||
|
||||
async fn run(&self) -> Result<()> {
|
||||
let control_object = self.control_object();
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
|
||||
return Ok(());
|
||||
};
|
||||
let guard = Arc::new(guard);
|
||||
let now = now_unix();
|
||||
match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
|
||||
Ok(bytes) => {
|
||||
let control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
if control.bucket_incarnation != self.bucket_incarnation
|
||||
|| control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
|
||||
{
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
if control.next_attempt_at > now {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
|
||||
let mut control = ReconcileControl {
|
||||
bucket_incarnation: self.bucket_incarnation,
|
||||
quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
|
||||
next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
|
||||
};
|
||||
// Persist the throttle before work, including bootstrap and failed scans.
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
let result = self.scan_and_publish(&guard).await;
|
||||
control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
result
|
||||
}
|
||||
|
||||
/// Take a short authoritative configuration snapshot without recursively
|
||||
/// acquiring a reader behind a queued metadata/lifecycle writer.
|
||||
async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
|
||||
let guard = snapshot
|
||||
.metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
|
||||
.ok_or(StorageError::PreconditionFailed)?;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(self.bucket_incarnation),
|
||||
object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
};
|
||||
let (quota, incarnation, revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
|
||||
.await?;
|
||||
if incarnation != self.bucket_incarnation
|
||||
|| revision != self.quota_revision
|
||||
|| !quota.as_ref().is_some_and(|quota| {
|
||||
quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
})
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
|
||||
let ledger_object = ledger_object(&self.bucket);
|
||||
let expected_revision = {
|
||||
// Lock order: reconcile owner, lifecycle, metadata, ledger.
|
||||
// Request paths never wait for the reconcile owner.
|
||||
let _configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(mut ledger) => {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// Seed a non-nil token even for a legacy ledger. An old
|
||||
// node drops this field on write and cannot pass the CAS.
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
|
||||
Some(ledger.writer_revision)
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
|
||||
// Only the independent reconcile owner is held during this whole-bucket
|
||||
// walk. Concurrent reserve/settle/reap writes invalidate its result.
|
||||
let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
|
||||
let configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(ledger) => {
|
||||
expected_revision == Some(ledger.writer_revision)
|
||||
&& !ledger.writer_revision.is_nil()
|
||||
&& ledger.reservations.is_empty()
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
if owner.is_lock_lost()
|
||||
|| !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
|
||||
{
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
|
||||
async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
|
||||
ledger.reconcile_required = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
|
||||
@@ -2136,7 +1935,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
|
||||
async fn save_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
ledger_object: &str,
|
||||
ledger: &mut QuotaLedger,
|
||||
ledger: &QuotaLedger,
|
||||
ledger_guard: &Arc<NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
if ledger_guard.is_lock_lost() {
|
||||
@@ -2160,71 +1959,9 @@ async fn save_ledger_locked(
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
ledger.writer_revision = Uuid::new_v4();
|
||||
save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
|
||||
}
|
||||
|
||||
/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
|
||||
/// The production reaper, ownership fence, scan and compare-and-swap still run.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
|
||||
let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let active = active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.contains(&key);
|
||||
if !active {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StorageError::SlowDown)?;
|
||||
reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
|
||||
let object = job.control_object();
|
||||
{
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match read_config_no_lock(Arc::clone(&store), &object).await {
|
||||
Ok(bytes) => {
|
||||
let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
|
||||
job.save_control(&object, &control, &guard).await?;
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
job.run().await
|
||||
}
|
||||
|
||||
/// Bootstrap a test bucket through the real background reconciliation path.
|
||||
/// Fixtures call this after enabling V1 quota, before issuing writes whose
|
||||
/// contract is independent of the initial retryable admission response.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
if !quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
|
||||
.run()
|
||||
.await?;
|
||||
match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
|
||||
Some(_) => Ok(()),
|
||||
None => Err(StorageError::SlowDown),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn fail_next_quota_ledger_save_for_test() {
|
||||
@@ -2266,434 +2003,6 @@ mod tests {
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ReconcilePauseStage {
|
||||
BeforeScan,
|
||||
BeforePublish,
|
||||
}
|
||||
|
||||
struct ReconcilePause {
|
||||
bucket: String,
|
||||
stage: ReconcilePauseStage,
|
||||
armed: std::sync::atomic::AtomicBool,
|
||||
scans: std::sync::atomic::AtomicUsize,
|
||||
fail_scan: std::sync::atomic::AtomicBool,
|
||||
reached: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
|
||||
|
||||
struct ReconcilePauseGuard(Arc<ReconcilePause>);
|
||||
|
||||
impl ReconcilePauseGuard {
|
||||
fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
|
||||
let pause = Arc::new(ReconcilePause {
|
||||
bucket: bucket.to_owned(),
|
||||
stage,
|
||||
armed: true.into(),
|
||||
scans: 0.into(),
|
||||
fail_scan: false.into(),
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
|
||||
Self(pause)
|
||||
}
|
||||
|
||||
async fn reached(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
|
||||
.await
|
||||
.expect("reconciliation reached barrier");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReconcilePauseGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.release.notify_one();
|
||||
let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
|
||||
if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
|
||||
use std::sync::atomic::Ordering;
|
||||
let hook = RECONCILE_PAUSE
|
||||
.lock()
|
||||
.expect("reconcile hooks")
|
||||
.as_ref()
|
||||
.filter(|hook| hook.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(hook) = hook {
|
||||
if stage == ReconcilePauseStage::BeforeScan {
|
||||
hook.scans.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
|
||||
return Err(StorageError::other("injected quota usage failure"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
|
||||
use crate::object_api::PutObjReader;
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let mut seed = PutObjReader::from_vec(vec![1; 10]);
|
||||
store
|
||||
.put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing usage");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("enable quota");
|
||||
let (_, incarnation, revision) = store
|
||||
.bucket_quota_and_incarnation_from_disk(&bucket)
|
||||
.await
|
||||
.expect("quota identity");
|
||||
(directories, ReconcileJob::new(store, bucket, incarnation, revision))
|
||||
}
|
||||
|
||||
async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
|
||||
begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
|
||||
.await?
|
||||
.reserve(old, new)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("ledger")
|
||||
}
|
||||
|
||||
async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
|
||||
let key = (
|
||||
job.store.id,
|
||||
job.bucket.clone(),
|
||||
job.bucket_incarnation,
|
||||
job.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while active_reconciliations().lock().expect("active workers").contains(&key) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned reconciliation finished");
|
||||
}
|
||||
|
||||
async fn expire_reconciliation_backoff(job: &ReconcileJob) {
|
||||
drain_reconciliation_scheduler(job).await;
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
|
||||
let mut control: ReconcileControl =
|
||||
serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
|
||||
.expect("control JSON");
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("control fence");
|
||||
job.save_control(&object, &control, &guard).await.expect("advance test clock");
|
||||
}
|
||||
|
||||
async fn wait_for_reconcile_owner(job: &ReconcileJob) {
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
|
||||
.await
|
||||
.expect("worker finished")
|
||||
.expect("owner guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
|
||||
"unknown usage cannot admit a reservation"
|
||||
);
|
||||
pause.reached().await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"bootstrap must not publish zero usage"
|
||||
);
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
|
||||
// Bypass the local scheduler, exercising the cross-node ownership boundary.
|
||||
tokio::time::timeout(Duration::from_secs(2), job.run())
|
||||
.await
|
||||
.expect("contender must not wait for a scan")
|
||||
.expect("another owner is active");
|
||||
let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
|
||||
.await
|
||||
.expect("scan does not retain lifecycle readers")
|
||||
.expect("lifecycle writer");
|
||||
let metadata = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
|
||||
)
|
||||
.await
|
||||
.expect("scan does not retain metadata readers")
|
||||
.expect("metadata writer");
|
||||
drop(metadata);
|
||||
drop(lifecycle);
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
let ledger = read_test_ledger(&job).await;
|
||||
assert_eq!(ledger.accounted_usage, 10);
|
||||
assert!(ledger.reservations.is_empty());
|
||||
assert!(!ledger.writer_revision.is_nil());
|
||||
for _ in 0..3 {
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
|
||||
job.run().await.expect("persisted cooldown");
|
||||
}
|
||||
drain_reconciliation_scheduler(&job).await;
|
||||
assert_eq!(
|
||||
pause.0.scans.load(Ordering::SeqCst),
|
||||
1,
|
||||
"denied requests cannot repeatedly scan a full bucket"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
job.run().await.expect("bootstrap");
|
||||
for mutation in ["abort", "commit", "reap", "legacy"] {
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
let initial = read_test_ledger(&job).await.writer_revision;
|
||||
if mutation == "legacy" {
|
||||
let object = ledger_object(&job.bucket);
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
|
||||
let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
|
||||
document.as_object_mut().expect("ledger map").remove("writer_revision");
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("ledger fence");
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(
|
||||
Arc::clone(&job.store),
|
||||
&object,
|
||||
serde_json::to_vec(&document).expect("old node JSON"),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("old node drops new field");
|
||||
} else {
|
||||
let mut reservation = quota_reservation(&job, 10, 10)
|
||||
.await
|
||||
.expect("zero-growth admission while scanning");
|
||||
let reserved = read_test_ledger(&job).await.writer_revision;
|
||||
assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
|
||||
reservation.mark_commit_started().await.expect("commit marker");
|
||||
assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
|
||||
match mutation {
|
||||
"abort" => reservation.abort().await,
|
||||
"commit" => reservation.commit().await,
|
||||
"reap" => {
|
||||
reservation.defer_after_fence();
|
||||
reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("reap orphan");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let written = read_test_ledger(&job).await;
|
||||
assert!(written.reservations.is_empty());
|
||||
assert_ne!(written.writer_revision, initial);
|
||||
pause.0.release.notify_one();
|
||||
worker.await.expect("worker join").expect("stale scan discarded");
|
||||
let current = read_test_ledger(&job).await;
|
||||
assert_eq!(
|
||||
current.writer_revision, written.writer_revision,
|
||||
"{mutation}: stale usage must not replace a concurrent writer"
|
||||
);
|
||||
assert_eq!(current.accounted_usage, written.accounted_usage);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
pause.0.fail_scan.store(true, Ordering::SeqCst);
|
||||
pause.0.release.notify_one();
|
||||
assert!(job.run().await.is_err());
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
job.run().await.expect("failure cooldown");
|
||||
assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
|
||||
pause.0.fail_scan.store(false, Ordering::SeqCst);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("recover after failure");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("delete stored usage");
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
|
||||
"stale usage remains conservative during cooldown"
|
||||
);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("refresh deleted usage");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
|
||||
quota_reservation(&job, 0, 1)
|
||||
.await
|
||||
.expect("delete must not leave bucket permanently full")
|
||||
.abort()
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let caller = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel caller").is_cancelled());
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
|
||||
)
|
||||
.await
|
||||
.expect("quota changes while scanning");
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"old config cannot publish bootstrap usage"
|
||||
);
|
||||
drop(pause);
|
||||
let (_, incarnation, revision) = job
|
||||
.store
|
||||
.bucket_quota_and_incarnation_from_disk(&job.bucket)
|
||||
.await
|
||||
.expect("current configuration");
|
||||
let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
let caller = tokio::spawn({
|
||||
let job = current.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel new caller").is_cancelled());
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(¤t).await;
|
||||
assert!(
|
||||
read_test_ledger(¤t).await.matches(incarnation, revision),
|
||||
"owned scan publishes after its waiter is canceled"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
|
||||
use crate::storage_api_contracts::bucket::DeleteBucketOptions;
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("empty old bucket");
|
||||
job.store
|
||||
.delete_bucket(&job.bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("retire bucket while scan is paused");
|
||||
job.store
|
||||
.make_bucket(&job.bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("replacement bucket");
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
|
||||
)
|
||||
.await
|
||||
.expect("replacement quota");
|
||||
pause.0.release.notify_one();
|
||||
assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
drop(pause);
|
||||
initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
|
||||
.await
|
||||
.expect("replacement can bootstrap independently");
|
||||
let replacement = read_test_ledger(&job).await;
|
||||
assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
|
||||
assert_eq!(replacement.accounted_usage, 0);
|
||||
}
|
||||
|
||||
fn ledger(accounted_usage: u64) -> QuotaLedger {
|
||||
QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
|
||||
}
|
||||
@@ -2781,7 +2090,8 @@ mod tests {
|
||||
.expect("persist crash-left reservation fixture");
|
||||
|
||||
assert!(matches!(
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await,
|
||||
Err(StorageError::PartMissingOrCorrupt)
|
||||
));
|
||||
assert!(matches!(
|
||||
@@ -2799,10 +2109,10 @@ mod tests {
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
let replacement =
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await
|
||||
.expect("new quota admission must not inherit a fresh stale reservation");
|
||||
assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
|
||||
assert!(replacement.reservations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::{
|
||||
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Entire SDK call, including non-streaming response bodies and retries.
|
||||
pub operation_timeout: Option<Duration>,
|
||||
/// How many wire requests one logical call may cost. Every consumer
|
||||
/// declares it; see [`RemoteS3RetryPolicy`].
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct UploadResponseDeadline(pub Duration);
|
||||
|
||||
impl Intercept for UploadResponseDeadline {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUploadResponseDeadline"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response_mut();
|
||||
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
|
||||
inner: Some(inner),
|
||||
deadline: Box::pin(tokio::time::sleep(self.0)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct UploadResponseBody {
|
||||
inner: Option<SdkBody>,
|
||||
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl http_body::Body for UploadResponseBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = BoxError;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
use std::future::Future as _;
|
||||
use std::task::Poll;
|
||||
if self.inner.is_none() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// The absolute response budget never renews on data, empty frames,
|
||||
// or trailers. A retry gets its own bounded response window.
|
||||
if self.deadline.as_mut().poll(cx).is_ready() {
|
||||
self.inner.take();
|
||||
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"replication response body deadline exceeded",
|
||||
)
|
||||
.into())));
|
||||
}
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
let result = std::pin::Pin::new(inner).poll_frame(cx);
|
||||
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
|
||||
self.inner.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
if let Some(operation_timeout) = spec.operation_timeout {
|
||||
timeouts = timeouts.operation_timeout(operation_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
@@ -753,7 +674,6 @@ mod tests {
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: None,
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
@@ -1107,16 +1027,11 @@ mod tests {
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
spec.operation_timeout = Some(Duration::from_secs(12));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
assert_eq!(
|
||||
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
|
||||
Some(Duration::from_secs(12))
|
||||
);
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
|
||||
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
@@ -117,10 +116,6 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
|
||||
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1581,7 +1462,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A source generation already queued or being uploaded is not driven a
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -2404,47 +2314,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name, version and source generation. `queue_replica_task` admits a generation only once
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct ReplicationInFlightKey {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<uuid::Uuid>,
|
||||
generation: ReplicationGenerationSnapshot,
|
||||
}
|
||||
|
||||
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
|
||||
fn from(info: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: info.bucket.clone(),
|
||||
object: info.name.clone(),
|
||||
version_id: info.version_id,
|
||||
generation: info.replication_generation.clone(),
|
||||
}
|
||||
}
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert(ReplicationInFlightKey::from(ri))
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&ReplicationInFlightKey::from(ri));
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2457,21 +2348,26 @@ impl ReplicationInFlight {
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicationInFlightKey,
|
||||
key: ReplicateObjectInfo,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicationInFlightKey::from(ri),
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.lock().remove(&self.key);
|
||||
self.in_flight.finish(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3525,8 +3421,6 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3594,24 +3488,6 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3675,16 +3551,6 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3806,12 +3672,7 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
Err(EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -4068,48 +3929,6 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let initial = ReplicateObjectInfo {
|
||||
bucket: "pending-tags-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = ReplicateObjectInfo {
|
||||
op_type: ReplicationType::Metadata,
|
||||
user_tags: String::new(),
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..initial.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
|
||||
assert_eq!(pool.in_flight.len(), 1);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
for expected in [&initial, &metadata] {
|
||||
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
|
||||
panic!("expected object replication");
|
||||
};
|
||||
assert_eq!(queued.replication_generation, expected.replication_generation);
|
||||
}
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
@@ -4352,8 +4171,6 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4404,103 +4221,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5584,8 +5304,6 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6550,108 +6268,6 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
source_already_purged
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
@@ -6609,7 +6604,6 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
timeouts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -259,10 +259,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
||||
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
@@ -274,14 +270,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// New public keys must remain representable in an unencoded XML listing.
|
||||
/// Existing keys still use the read/delete validators so they can be removed.
|
||||
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
|
||||
object
|
||||
.chars()
|
||||
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
|
||||
}
|
||||
|
||||
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{Instrument, debug, error, warn};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
@@ -224,8 +224,6 @@ pub struct ListPathRawOptions {
|
||||
pub recursive: bool,
|
||||
pub incl_deleted: bool,
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
@@ -262,8 +260,6 @@ impl Clone for ListPathRawOptions {
|
||||
recursive: self.recursive,
|
||||
incl_deleted: self.incl_deleted,
|
||||
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
@@ -293,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
recursive: opts.recursive,
|
||||
incl_deleted: opts.incl_deleted,
|
||||
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: opts.hidden_entry_budget,
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
@@ -307,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
/// Inclusive sorted boundary proved by every reader before a bounded stop.
|
||||
pub scan_cut: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
@@ -371,15 +363,10 @@ async fn list_path_raw_inner(
|
||||
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
|
||||
|
||||
let cancel_rx = CancellationToken::new();
|
||||
let walk_options = Arc::new(walk_dir_options(&opts));
|
||||
|
||||
for (disk_idx, disk) in opts.disks.iter().enumerate() {
|
||||
let opdisk = disk.clone();
|
||||
let walk_options = walk_options.clone();
|
||||
#[cfg(test)]
|
||||
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
|
||||
#[cfg(test)]
|
||||
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
|
||||
let opts_clone = opts.clone();
|
||||
let fallback_claim_tracker = fallback_claim_tracker.clone();
|
||||
let fds_clone = fds.clone();
|
||||
#[cfg(test)]
|
||||
@@ -389,17 +376,17 @@ async fn list_path_raw_inner(
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64 * 1024);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &test_disk_io_context_observed {
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = test_reader_behavior {
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
TestReaderBehavior::Entries(entries) => {
|
||||
@@ -456,6 +443,8 @@ async fn list_path_raw_inner(
|
||||
None
|
||||
};
|
||||
|
||||
let wakl_opts = walk_dir_options(&opts_clone);
|
||||
|
||||
let mut need_fallback = false;
|
||||
let mut last_err = None;
|
||||
#[cfg(test)]
|
||||
@@ -470,7 +459,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
if !need_fallback && let Some(disk) = opdisk {
|
||||
let primary_walk_started = std::time::Instant::now();
|
||||
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
Ok(_res) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_primary",
|
||||
@@ -494,8 +483,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -506,8 +495,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -609,8 +598,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing_for_path",
|
||||
error = ?err,
|
||||
@@ -621,8 +610,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing",
|
||||
error = ?err,
|
||||
@@ -637,7 +626,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
let fallback_walk_started = std::time::Instant::now();
|
||||
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
|
||||
Ok(_r) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_fallback",
|
||||
@@ -663,8 +652,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -675,8 +664,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -698,13 +687,12 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
});
|
||||
jobs.spawn(producer.in_current_span());
|
||||
}));
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
let mut revjobs = JoinSet::new();
|
||||
let merge = async move {
|
||||
revjobs.spawn(async move {
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
@@ -734,8 +722,6 @@ async fn list_path_raw_inner(
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
let mut scan_cut = None;
|
||||
let mut last_scanned_key: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -747,7 +733,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -807,12 +793,7 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
if r.limit_reached() == Some(true) {
|
||||
limit_reached = true;
|
||||
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
|
||||
DiskError::other("bounded listing reader ended without a scan boundary")
|
||||
})?);
|
||||
}
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -886,7 +867,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -945,7 +926,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -995,14 +976,6 @@ async fn list_path_raw_inner(
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
// The next head from a bounded reader is unknown. The previous
|
||||
// globally consumed group is a safe cut; consuming another reader's
|
||||
// later head would turn this absence of evidence into a skipped key.
|
||||
// Check quorum failures above before accepting the bounded result.
|
||||
if limit_reached {
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0
|
||||
@@ -1010,7 +983,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -1021,7 +994,6 @@ async fn list_path_raw_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
last_scanned_key = Some(current.name.clone());
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
if !r.skip_current() {
|
||||
@@ -1033,7 +1005,7 @@ async fn list_path_raw_inner(
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
@@ -1057,14 +1029,13 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ListPathRawOutcome { limit_reached, scan_cut })
|
||||
};
|
||||
revjobs.spawn(merge.in_current_span());
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
});
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
let Some(revjob) = revjobs.join_next().await else {
|
||||
@@ -1228,24 +1199,6 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
bucket: "versioned".to_owned(),
|
||||
hidden_entry_budget: 100,
|
||||
pending_prefix: Some("pending/".to_owned()),
|
||||
forward_to: Some("pending/0099".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
// Primary and fallback readers both construct their walk from the
|
||||
// per-job clone, rather than directly from the request options.
|
||||
let job = request.clone();
|
||||
let walk = walk_dir_options(&job);
|
||||
assert_eq!(walk.hidden_entry_budget, 100);
|
||||
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
|
||||
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
@@ -1296,163 +1249,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let baseline = Arc::strong_count(&disk);
|
||||
|
||||
for primary_count in [1, 4, 16] {
|
||||
for fallback_count in [0, 2] {
|
||||
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let behaviors = entered
|
||||
.iter()
|
||||
.zip(&stopped)
|
||||
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
|
||||
entered: entered.clone(),
|
||||
stopped: stopped.clone(),
|
||||
})
|
||||
.collect();
|
||||
let cancel = CancellationToken::new();
|
||||
let listing = tokio::spawn(list_path_raw(
|
||||
cancel.clone(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk.clone()); primary_count],
|
||||
fallback_disks: vec![Some(disk.clone()); fallback_count],
|
||||
min_disks: primary_count,
|
||||
test_reader_behaviors: behaviors,
|
||||
peek_timeout: Some(Duration::from_secs(60)),
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
for signal in &entered {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("every producer must enter the real fanout before counting references");
|
||||
}
|
||||
|
||||
// The merge owns the configured primary/fallback slots, each producer
|
||||
// owns its primary, and the shared fallback queue owns each fallback.
|
||||
let expected = baseline + 2 * (primary_count + fallback_count);
|
||||
let retained = Arc::strong_count(&disk);
|
||||
cancel.cancel();
|
||||
timeout(Duration::from_secs(2), listing)
|
||||
.await
|
||||
.expect("cancellation must stop the listing")
|
||||
.expect("listing task must join")
|
||||
.expect("external cancellation must finish cleanly");
|
||||
for signal in &stopped {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("cancellation must drop every producer");
|
||||
}
|
||||
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
|
||||
assert_eq!(
|
||||
retained, expected,
|
||||
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn list_path_raw_producer_and_merge_preserve_request_span() {
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt};
|
||||
|
||||
struct ProducerSpan(Arc<AtomicBool>);
|
||||
#[derive(Default)]
|
||||
struct StateField(bool);
|
||||
impl tracing::field::Visit for StateField {
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
|
||||
}
|
||||
}
|
||||
impl<S> Layer<S> for ProducerSpan
|
||||
where
|
||||
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
|
||||
let mut state = StateField::default();
|
||||
event.record(&mut state);
|
||||
if state.0
|
||||
&& context
|
||||
.lookup_current()
|
||||
.is_some_and(|span| span.name() == "flink_list_request")
|
||||
{
|
||||
self.0.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
|
||||
let producer_observed = Arc::new(AtomicBool::new(false));
|
||||
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let error = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "missing-bucket".to_owned(),
|
||||
min_disks: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request.clone())
|
||||
.await
|
||||
.expect_err("missing bucket must fail the real disk producer");
|
||||
assert_eq!(error, DiskError::VolumeNotFound);
|
||||
assert!(
|
||||
producer_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"disk error must retain the request span"
|
||||
);
|
||||
|
||||
let merge_observed = Arc::new(AtomicBool::new(false));
|
||||
let merge_sink = merge_observed.clone();
|
||||
let request_id = request.id();
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
|
||||
agreed: Some(Box::new(move |_| {
|
||||
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
|
||||
Box::pin(async { Ok(()) })
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request)
|
||||
.await
|
||||
.expect("merge should deliver the test entry");
|
||||
assert!(
|
||||
merge_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"merge callback must retain the request span"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
@@ -1520,7 +1316,6 @@ mod tests {
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
@@ -1588,13 +1590,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)?;
|
||||
// A body-bound request also carries a usable v2 signature. Consume both
|
||||
// nonces so stripping the replay-scope headers cannot authorize it again.
|
||||
if content_sha256 != UNSIGNED_PAYLOAD {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
|
||||
}
|
||||
Ok(())
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -2585,43 +2581,6 @@ mod tests {
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_consumes_the_body_bound_v2_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "node-a:9000";
|
||||
let path = "/node_service.NodeService/RecoverDeleteTransaction";
|
||||
let digest = "a".repeat(64);
|
||||
for v3_first in [true, false] {
|
||||
let mut headers =
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
|
||||
.expect("body-bound v2 headers should build");
|
||||
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, &digest, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let mut downgraded = headers.clone();
|
||||
for header in [
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
] {
|
||||
downgraded.remove(header);
|
||||
}
|
||||
let (first, replay) = if v3_first {
|
||||
(&headers, &downgraded)
|
||||
} else {
|
||||
(&downgraded, &headers)
|
||||
};
|
||||
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
|
||||
.expect("the first delivery must be accepted");
|
||||
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
|
||||
.expect_err("a mutation cannot replay through the other signature generation");
|
||||
assert_eq!(error.to_string(), "RPC request replay detected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -2014,21 +2014,9 @@ impl PeerRestClient {
|
||||
}
|
||||
result
|
||||
})
|
||||
.await;
|
||||
self.settle_bucket_metadata_reload(bucket, result).await
|
||||
}
|
||||
|
||||
async fn settle_bucket_metadata_reload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
|
||||
) -> Result<()> {
|
||||
match result {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
// A local operation deadline does not establish that the peer is
|
||||
// unreachable. In particular, it must not gate IAM revocations.
|
||||
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
|
||||
}
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
@@ -4765,31 +4753,6 @@ mod tests {
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
|
||||
let client = test_peer_client();
|
||||
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("slow-policy", elapsed)
|
||||
.await
|
||||
.expect_err("local budget expires");
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
assert!(
|
||||
!client.offline.load(Ordering::Acquire),
|
||||
"local work must not suppress subsequent IAM RPCs"
|
||||
);
|
||||
client
|
||||
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
|
||||
.await
|
||||
.expect("peer reply");
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
|
||||
.await
|
||||
.expect_err("real transport failure");
|
||||
assert!(error.to_string().contains("connection refused"));
|
||||
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
|
||||
MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -1082,10 +1083,7 @@ impl RemoteDisk {
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
}
|
||||
let settle_timeout = if matches!(
|
||||
&operation,
|
||||
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
|
||||
) {
|
||||
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
|
||||
Some(crate::disk::get_delete_recovery_timeout()?)
|
||||
} else {
|
||||
None
|
||||
@@ -1640,7 +1638,6 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1712,7 +1709,6 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1783,7 +1779,6 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2015,7 +2010,6 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2030,7 +2024,6 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2460,6 +2453,239 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
/// Prepare a remote token without retrying the RPC. It owns the bucket
|
||||
/// publication fence but does not enter the object metadata queue yet.
|
||||
pub async fn reserve_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Use the distinct short-lease RPC; an old peer rejects it before allocating a token.
|
||||
pub(crate) async fn reserve_object_metadata_short(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata_with_protocol(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
short: bool,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"reserve_object_metadata",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let result = if short {
|
||||
client.reserve_object_metadata_short(request).await
|
||||
} else {
|
||||
client.reserve_object_metadata(request).await
|
||||
};
|
||||
let response = result
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
if response.protocol_version != if short { 3 } else { 2 } {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
Uuid::from_slice(&response.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or(DiskError::Unexpected)
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
|
||||
/// result before allowing any subsequent unprotected mutation.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
self.update_object_metadata_reservation(bucket, object, token, timeout, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
self.update_object_metadata_reservation(bucket, object, token, timeout, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_object_metadata_reservation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: Duration,
|
||||
renew_only: bool,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"activate_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationActivateRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
renew_only,
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation activation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.activate_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed remote reservation. False cannot prove that an
|
||||
/// earlier claim did not start a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"cancel_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationCancelRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.cancel_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(response.success)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Publish a staged user-object write using a remote one-shot reservation.
|
||||
/// The caller must retain the bucket lifecycle fence until the tail drains.
|
||||
pub async fn rename_data_with_metadata_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
token: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_reserved_data",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let rename = RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: Bytes::new(),
|
||||
};
|
||||
let mut request = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(rename),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reserved rename request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.rename_reserved_data(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -2719,54 +2945,6 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
|
||||
let expected_disk = self
|
||||
.id
|
||||
.lock()
|
||||
.await
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query = MultipartIndexRpcRequest::new(bucket, marker);
|
||||
let endpoint = self.endpoint.to_string();
|
||||
let canonical = query.canonical_request(&endpoint)?;
|
||||
let opts = serde_json::to_string(&query)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DiskInfoRequest {
|
||||
disk: endpoint.clone(),
|
||||
opts: opts.clone(),
|
||||
});
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = client.disk_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// An old server ignores the additive query and returns ordinary DiskInfo.
|
||||
// It cannot satisfy the dedicated reply schema or the fresh challenge.
|
||||
let reply: MultipartIndexRpcReply =
|
||||
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
|
||||
reply.validate(&query, expected_disk)?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&query.canonical_response(&endpoint, &reply.page)?,
|
||||
&reply.proof,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
Ok(reply.page)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
@@ -4350,38 +4528,6 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
// Older peers reject this variant. Do not retry through the unbounded protocol;
|
||||
// they are reported as unsupported participants (rolling-upgrade contract).
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::DiscoverBounded {
|
||||
object: object.to_owned(),
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4417,62 +4563,6 @@ impl DiskAPI for RemoteDisk {
|
||||
_ => Err(Error::other("unexpected delete discovery reply")),
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::InspectBounded {
|
||||
object: object.to_owned(),
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4993,113 +5083,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("test peer must bind");
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk"),
|
||||
);
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let prefix = directory.path().join("bucket/prefix");
|
||||
std::fs::create_dir(&prefix).expect("prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
let fixture = Arc::new(TestBoundedDiscovery {
|
||||
disk: Arc::clone(&disk),
|
||||
response_gate: TestFormatReadGate::default(),
|
||||
});
|
||||
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
|
||||
.await
|
||||
.expect("disk completed bounded scan");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("caller cancelled").is_cancelled());
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
fixture.response_gate.release.cancel();
|
||||
peer.stop().await;
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
assert!(prefix.join("child-63").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
remote
|
||||
.set_disk_id(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("expected disk identity");
|
||||
assert!(matches!(
|
||||
remote.list_multipart_index_roots("bucket", None).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
assert_eq!(
|
||||
peer.peer.disk_info_calls.load(Ordering::Acquire),
|
||||
1,
|
||||
"exercise the actual legacy JSON handler"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(
|
||||
remote
|
||||
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
|
||||
.await
|
||||
.expect_err("old peer cannot prove a bounded inventory"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
remote
|
||||
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
|
||||
.await
|
||||
.expect_err("old peer cannot inspect with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.settle_delete_transaction_bounded(
|
||||
"bucket",
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "prefix".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: [0; 32],
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: 65536,
|
||||
},
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot settle with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods"),
|
||||
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5360,6 +5343,84 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_metadata_reservation_rejects_old_peer_without_legacy_rpc() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata_short("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec!["/node_service.NodeService/ReserveObjectMetadataShort"],
|
||||
"an old peer must never receive the longer-lived v2 request"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let token = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.renew_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote.cancel_object_metadata_reservation(token).await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.rename_data_with_metadata_reservation(
|
||||
crate::disk::RUSTFS_META_TMP_BUCKET,
|
||||
"staged",
|
||||
&sample_rename_data_file_info(),
|
||||
"bucket",
|
||||
"object",
|
||||
token,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec![
|
||||
"/node_service.NodeService/ReserveObjectMetadata",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/CancelObjectMetadataReservation",
|
||||
"/node_service.NodeService/RenameReservedData",
|
||||
],
|
||||
"reservation operations must never downgrade to legacy methods",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5992,16 +6053,9 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestBoundedDiscovery {
|
||||
disk: Arc<crate::disk::local::LocalDisk>,
|
||||
response_gate: TestFormatReadGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
@@ -6018,7 +6072,6 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
bounded_discovery: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -6118,56 +6171,6 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/RecoverDeleteTransaction"
|
||||
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
|
||||
{
|
||||
#[derive(Clone)]
|
||||
struct BoundedSvc(Arc<TestBoundedDiscovery>);
|
||||
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
|
||||
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
|
||||
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
|
||||
let fixture = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let request = request.into_inner();
|
||||
let operation: DeleteRecoveryRpcOperation =
|
||||
rustfs_protos::decode_delete_recovery_payload(&request.operation)
|
||||
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
|
||||
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
|
||||
return Err(Status::invalid_argument("unexpected fallback"));
|
||||
};
|
||||
let page = fixture
|
||||
.disk
|
||||
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
|
||||
.await
|
||||
.map_err(|_| Status::internal("bounded discovery failed"))?;
|
||||
// Lose the response only after the real disk operation has completed.
|
||||
fixture.response_gate.entered.notify_one();
|
||||
fixture.response_gate.release.cancelled().await;
|
||||
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
|
||||
.map_err(|_| Status::internal("encode bounded reply"))?;
|
||||
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
|
||||
success: true,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
result: result.into(),
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
let fixture = self
|
||||
.peer
|
||||
.bounded_discovery
|
||||
.lock()
|
||||
.expect("bounded fixture")
|
||||
.clone()
|
||||
.expect("configured fixture");
|
||||
Box::pin(async move {
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec);
|
||||
Ok(grpc.unary(BoundedSvc(fixture), request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/ReadAll" => {
|
||||
#[derive(Clone)]
|
||||
struct ReadAllSvc(AuthenticatedReadPeer);
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::cluster::rpc::client::{
|
||||
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
@@ -33,17 +32,16 @@ use rustfs_protos::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
@@ -51,38 +49,18 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
|
||||
|
||||
/// Own admission before connecting as well as while an RPC is in flight.
|
||||
struct LockRequestGuard<'a> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl LockRequestGuard<'_> {
|
||||
fn into_admission(mut self) -> Option<LockRequestAdmission> {
|
||||
self.admission.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LockRequestGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.client.release_lock_request(self.op, self.admission.take());
|
||||
}
|
||||
}
|
||||
|
||||
struct RpcCleanup<'a, T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestGuard<'a>>,
|
||||
}
|
||||
|
||||
impl<'a, T> RpcCleanup<'a, T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
@@ -91,25 +69,6 @@ impl<'a, T> RpcCleanup<'a, T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
|
||||
struct InFlightRpc<'a, T: Send + 'static> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
resource_summary: &'a str,
|
||||
future: Option<RpcFuture<T>>,
|
||||
cleanup: Option<RpcCleanup<'a, T>>,
|
||||
}
|
||||
|
||||
impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(future) = self.future.take()
|
||||
&& let Some(cleanup) = self.cleanup.take()
|
||||
{
|
||||
self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
@@ -178,9 +137,6 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(health) = peers.get_mut(addr) {
|
||||
return update(health);
|
||||
}
|
||||
update(peers.entry(addr.to_string()).or_default())
|
||||
}
|
||||
|
||||
@@ -202,15 +158,6 @@ fn reset_lock_peer_health_for_test(addr: &str) {
|
||||
.remove(addr);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
|
||||
static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
|
||||
PAUSES.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum EvictionTrigger {
|
||||
/// The caller's deadline expired while the stream was still open.
|
||||
@@ -498,10 +445,6 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
});
|
||||
// A successful lock RPC over a cached or fresh channel proves the peer is
|
||||
// reachable; reconcile the internode peer-health gauge so it does not stay
|
||||
// stale when no fresh dial fires (issue #9119).
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
@@ -559,14 +502,6 @@ impl RemoteClient {
|
||||
admission
|
||||
}
|
||||
|
||||
fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
|
||||
self.admit_lock_request(op).map(|admission| LockRequestGuard {
|
||||
client: self,
|
||||
op,
|
||||
admission: Some(admission),
|
||||
})
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -648,17 +583,6 @@ impl RemoteClient {
|
||||
trigger: EvictionTrigger,
|
||||
deadline: Duration,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
let pause = eviction_pauses_for_test()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.addr);
|
||||
#[cfg(test)]
|
||||
if let Some((entered, resume)) = pause {
|
||||
let _ = entered.send(());
|
||||
let _ = resume.await;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let cooldown = Self::eviction_cooldown();
|
||||
let liveness_window = Self::liveness_window(deadline);
|
||||
@@ -689,26 +613,22 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
|
||||
/// Keep an abandoned RPC running instead of cancelling its stream.
|
||||
/// Keep a timed-out RPC running instead of cancelling its stream.
|
||||
///
|
||||
/// Dropping the future sends `RST_STREAM`; under load those resets pile up
|
||||
/// in the server's pending-accept queue until it answers `GOAWAY
|
||||
/// too_many_resets` and kills every stream on the connection. A detached
|
||||
/// stream ends on its own within the internode RPC timeout, the number per
|
||||
/// peer is bounded, and a lock granted after its caller gave up is released.
|
||||
fn detach_rpc<T: Send + 'static>(
|
||||
fn detach_timed_out_rpc<T: Send + 'static>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
resource_summary: &str,
|
||||
future: RpcFuture<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
) {
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let admission = admission.and_then(LockRequestGuard::into_admission);
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
if health.detached_rpcs >= limit {
|
||||
@@ -719,29 +639,25 @@ impl RemoteClient {
|
||||
}
|
||||
});
|
||||
if !admitted {
|
||||
drop(future);
|
||||
handle.abort();
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "detached_budget_exhausted",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
limit,
|
||||
"Abandoned remote lock RPC cancelled"
|
||||
"Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
|
||||
);
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = future.await;
|
||||
let outcome = handle.await;
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
@@ -756,7 +672,6 @@ impl RemoteClient {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
late(response).await;
|
||||
@@ -768,31 +683,19 @@ impl RemoteClient {
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_failed",
|
||||
addr = %addr,
|
||||
op,
|
||||
tonic_code = ?status.code(),
|
||||
tonic_message = status.message(),
|
||||
"Abandoned remote lock RPC failed"
|
||||
"Detached remote lock RPC failed after its caller timed out"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
Err(join_error) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_panicked",
|
||||
addr = %addr,
|
||||
op,
|
||||
"Abandoned remote lock RPC panicked"
|
||||
);
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -880,28 +783,16 @@ impl RemoteClient {
|
||||
self.release_late_release();
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_released",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
"Released remote locks granted after their caller stopped waiting"
|
||||
);
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_unreleased",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
@@ -915,47 +806,25 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
cleanup: RpcCleanup<T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let mut rpc = InFlightRpc {
|
||||
client: self,
|
||||
op,
|
||||
resource_summary,
|
||||
future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
|
||||
cleanup: Some(cleanup),
|
||||
};
|
||||
let outcome = match rpc.future.as_mut() {
|
||||
Some(future) => tokio::select! {
|
||||
// A response becoming ready on the deadline tick must not turn an
|
||||
// expired caller budget into a transport error and channel eviction.
|
||||
biased;
|
||||
_ = tokio::time::sleep(deadline) => None,
|
||||
outcome = future => Some(outcome),
|
||||
},
|
||||
None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
|
||||
};
|
||||
if outcome.is_some() {
|
||||
rpc.future.take();
|
||||
}
|
||||
let Some(RpcCleanup {
|
||||
late: _,
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
}) = rpc.cleanup.as_mut()
|
||||
else {
|
||||
return Err(LockError::internal("remote lock RPC has no cleanup owner"));
|
||||
};
|
||||
let token = admission.as_ref().and_then(|guard| guard.admission);
|
||||
let result = match outcome {
|
||||
Some(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, token);
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
Ok(response)
|
||||
}
|
||||
Some(Ok(Err(err))) => {
|
||||
Ok(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
@@ -988,11 +857,12 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
@@ -1053,32 +923,30 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
if transport_failure {
|
||||
// Ambiguous-grant cleanup owns its work before channel eviction can suspend.
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Some(Err(_)) => {
|
||||
Ok(Err(join_error)) => {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "rpc_panicked",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
"Remote lock RPC panicked"
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
);
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC future panicked")))
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
}
|
||||
None => {
|
||||
Err(_) => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
@@ -1110,17 +978,12 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
// Continue draining the abandoned stream while channel eviction waits.
|
||||
drop(rpc);
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
}
|
||||
};
|
||||
if rpc.future.is_none() {
|
||||
rpc.cleanup.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
|
||||
@@ -1211,7 +1074,7 @@ impl RemoteClient {
|
||||
#[async_trait]
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let admission = match self.admit_lock_request_guard("lock") {
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
@@ -1222,6 +1085,7 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1229,12 +1093,14 @@ impl LockClient for RemoteClient {
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
@@ -1276,7 +1142,7 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let admission = match self.admit_lock_request_guard("lock_batch") {
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
@@ -1290,6 +1156,7 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1303,12 +1170,14 @@ impl LockClient for RemoteClient {
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
@@ -1446,6 +1315,122 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
if !response.success {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
|
||||
Some(":0") => Some(false),
|
||||
Some(":1") => Some(true),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
@@ -1836,351 +1821,6 @@ mod tests {
|
||||
assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_connection_releases_request_admission() {
|
||||
ensure_test_rpc_secret();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
|
||||
let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
|
||||
for batch in [false, true] {
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
|
||||
});
|
||||
let mut acquisition = Box::pin(async {
|
||||
if batch {
|
||||
client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
|
||||
} else {
|
||||
client.acquire_lock(&request).await.map(|_| ())
|
||||
}
|
||||
});
|
||||
std::future::poll_fn(|cx| {
|
||||
assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
|
||||
std::task::Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
let pending = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
|
||||
assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
|
||||
|
||||
drop(acquisition);
|
||||
let cancelled = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
|
||||
assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
|
||||
let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
}
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-grant";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
Ok(grant_rx.await.expect("test decides when peer grants lock"))
|
||||
},
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"cancelled RPC uses bounded detached ownership"
|
||||
);
|
||||
grant_tx.send(true).expect("cancellation must preserve the running RPC");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up")
|
||||
.expect("cleanup result")
|
||||
);
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-failure";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("ambiguous cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
failure_rx.await.expect("test decides when transport fails");
|
||||
Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
failure_tx.send(()).expect("cancellation must preserve the running RPC");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("ambiguous grant must be cleaned up")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
|
||||
for (label, status) in [
|
||||
("transport", tonic::Status::cancelled("ambiguous transport failure")),
|
||||
("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
|
||||
] {
|
||||
let addr = format!("http://cancelled-completed-{label}");
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.clone(), (evicting_tx, resume_rx));
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx
|
||||
.send(())
|
||||
.expect("completed-failure cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move { Err::<(), _>(status) },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("completed error must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("cleanup must transfer before channel eviction suspends")
|
||||
.expect("cleanup signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(&addr).detached_rpcs,
|
||||
0,
|
||||
"completed RPC cannot consume a detached slot"
|
||||
);
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
|
||||
let addr = "http://same-tick-rpc-deadline";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let budget = Duration::from_millis(50);
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("same-tick cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
let result = client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
budget,
|
||||
async move {
|
||||
tokio::time::sleep(budget).await;
|
||||
Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"expired caller budget must retain timeout classification"
|
||||
);
|
||||
let health = lock_peer_health_for_test(addr);
|
||||
assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
|
||||
assert!(
|
||||
health.last_eviction.is_none(),
|
||||
"a recently serving peer keeps its channel after a caller timeout"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("same-tick ambiguous completion must trigger cleanup")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"same-tick completion returns detached admission"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
|
||||
let addr = "http://timeout-late-grant-during-eviction";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.to_string(), (evicting_tx, resume_rx));
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_millis(25),
|
||||
async move { Ok(grant_rx.await.expect("test controls the late grant")) },
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("timed-out RPC must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"eviction waits with a detached RPC owner"
|
||||
);
|
||||
grant_tx
|
||||
.send(true)
|
||||
.expect("timed-out stream remains available for a late grant");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up before eviction resumes")
|
||||
.expect("late cleanup result")
|
||||
);
|
||||
assert!(!caller.is_finished(), "channel eviction is still paused");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"late completion returns detached admission"
|
||||
);
|
||||
resume_tx.send(()).expect("resume channel eviction");
|
||||
let result = caller.await.expect("timed-out caller completes without panicking");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"caller preserves its RPC timeout result"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_success_polls_rpc_on_callers_task() {
|
||||
tokio::spawn(async {
|
||||
let client = RemoteClient::new("http://direct-poll-success".to_string());
|
||||
let caller_id = tokio::task::id();
|
||||
let rpc_id = client
|
||||
.execute_rpc(
|
||||
"ping",
|
||||
"health-lock-online",
|
||||
Duration::from_secs(1),
|
||||
async { Ok(tokio::task::id()) },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await
|
||||
.expect("successful RPC must return its result");
|
||||
assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
|
||||
})
|
||||
.await
|
||||
.expect("caller task must complete");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client = RemoteClient::new("http://direct-poll-panic".to_string());
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("panic cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc::<(), _>(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(1),
|
||||
async { panic!("test peer RPC panic") },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let result = caller.await.expect("RPC panic must not escape to its caller");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Internal { .. })),
|
||||
"RPC panic remains an internal lock failure"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("panic must trigger ambiguous-grant cleanup")
|
||||
.expect("cleanup signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -6838,47 +6839,11 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7269,8 +7234,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7325,8 +7288,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7340,10 +7301,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7357,36 +7314,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, selected_raw, mut selected) =
|
||||
let (_, _, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -14606,23 +14546,6 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14633,10 +14556,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != repair_data {
|
||||
if confirmed != *canonical {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -29559,53 +29482,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -317,9 +317,8 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn stop_endpoint_monitor_for_test(&self) {
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
@@ -656,6 +655,19 @@ impl Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -55,20 +55,6 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
|
||||
/// Completion may mean legacy fallback after a failed scan; callers must assert
|
||||
/// the expected index result separately. Remote disks have no local worker.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
|
||||
match disk.as_ref() {
|
||||
crate::disk::Disk::Local(local) => {
|
||||
local.disk.wait_for_multipart_index_bootstrap().await;
|
||||
Ok(())
|
||||
}
|
||||
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disk health status constants
|
||||
const DISK_HEALTH_OK: u32 = 0;
|
||||
const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
@@ -618,8 +604,69 @@ impl LocalDiskWrapper {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
metadata_reservation_token,
|
||||
..
|
||||
} = guards;
|
||||
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
|
||||
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
|
||||
(Some(reservation), None) => Some(reservation),
|
||||
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
// A missing token may already have been claimed by an earlier
|
||||
// request that is still publishing; do not claim it was safe.
|
||||
Err(DiskError::DiskOngoingReq) => {
|
||||
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
},
|
||||
(None, None) => None,
|
||||
};
|
||||
// A queued reservation must acquire its metadata lease before disk
|
||||
// admission. Otherwise a waiting background mutation could hold the
|
||||
// disk permit needed by the earlier lease owner.
|
||||
let metadata_reservation = match metadata_reservation {
|
||||
Some(reservation) => Some(reservation.wait().await),
|
||||
None => None,
|
||||
};
|
||||
// The claimed reservation stays in this cancellable future while disk
|
||||
// admission can wait. Only physical work moves into an owned task.
|
||||
let prechecked_permit = if metadata_reservation.is_some() {
|
||||
// Background admission is exclusive. Waiting for it while holding
|
||||
// this object's lease can deadlock an admitted mutation waiting for
|
||||
// the same lease. An already owned permit does not need to wait.
|
||||
if current_disk_io_context().is_some_and(|context| {
|
||||
context.class == DiskIoClass::Background
|
||||
&& !context
|
||||
.permit
|
||||
.as_ref()
|
||||
.is_some_and(|permit| permit.is_for(&self.io_admission))
|
||||
}) {
|
||||
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let preflight = self
|
||||
.with_native_resources(async {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.acquire_disk_io_permit().await
|
||||
})
|
||||
.await;
|
||||
match preflight {
|
||||
Ok(Ok(Some(permit))) => Some(permit),
|
||||
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The reserved metadata lease must outlive a cancelled RPC waiter.
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
|
||||
Some(Arc::new(external_guard))
|
||||
} else {
|
||||
external_guard
|
||||
};
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -638,27 +685,54 @@ impl LocalDiskWrapper {
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let result = operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
let rename = || async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = if let Some(reservation) = metadata_reservation {
|
||||
Box::pin(operation.disk.rename_data_observed_with_reservation(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
(namespace_owner, Some(reservation)),
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await
|
||||
};
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
};
|
||||
let result = if let Some(permit) = prechecked_permit {
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
operation
|
||||
.with_native_resources(operation.track_admitted_disk_operation(
|
||||
"rename_data",
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
rename().await
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::Write,
|
||||
true,
|
||||
))
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
.await?
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
operation
|
||||
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
|
||||
.await
|
||||
};
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
@@ -867,8 +941,6 @@ pub(crate) struct DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
|
||||
/// Caller API operations currently executing through the disk health wrapper.
|
||||
api_waiting: AtomicU32,
|
||||
/// Operations rejected because the disk was unavailable for the caller.
|
||||
@@ -896,8 +968,6 @@ impl Default for DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex::new(None),
|
||||
api_waiting: AtomicU32::new(0),
|
||||
total_errors_availability: AtomicU64::new(0),
|
||||
total_errors_timeout: AtomicU64::new(0),
|
||||
@@ -1591,20 +1661,6 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScopedDiskOperationTimeout {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
previous: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedDiskOperationTimeout {
|
||||
fn drop(&mut self) {
|
||||
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1630,6 +1686,81 @@ pub struct LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
/// Reserve the metadata mutation lock for one object on this disk instance.
|
||||
/// Unclaimed reservations expire after 60 seconds.
|
||||
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.reserve_object_metadata(bucket, object, timeout).await
|
||||
}
|
||||
|
||||
/// Register an object metadata reservation in FIFO order on this disk.
|
||||
/// The returned token may be claimed before its physical lease is available.
|
||||
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.enqueue_object_metadata(bucket, object).await
|
||||
}
|
||||
|
||||
/// Prepare an inert reservation token bound to this disk and object.
|
||||
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
self.prepare_object_metadata_with_owner(bucket, object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner_ttl(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
ttl: Duration,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
|
||||
}
|
||||
|
||||
/// Refresh an inert token while the caller is still staging the write.
|
||||
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.renew_object_metadata(bucket, object, token)
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
|
||||
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
|
||||
self.disk.cancel_object_metadata_reservation(token)
|
||||
}
|
||||
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
@@ -1946,23 +2077,6 @@ impl LocalDiskWrapper {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
|
||||
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
|
||||
// The metrics epoch follows this drive across clones and reconnects.
|
||||
// Its scoped override cannot shorten unrelated healthy drives' I/O.
|
||||
let previous = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.replace(timeout);
|
||||
ScopedDiskOperationTimeout {
|
||||
metrics: Arc::clone(&self.metrics),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -2457,7 +2571,6 @@ impl LocalDiskWrapper {
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2465,7 +2578,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
|
||||
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2481,7 +2594,6 @@ impl LocalDiskWrapper {
|
||||
transaction: uuid::Uuid,
|
||||
journal: &crate::disk::DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2489,14 +2601,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2639,6 +2744,7 @@ impl LocalDiskWrapper {
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
@@ -2650,6 +2756,7 @@ impl LocalDiskWrapper {
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
identity_prechecked: bool,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
@@ -2671,7 +2778,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if let Err(err) = self.check_disk_stale().await {
|
||||
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return Err(err);
|
||||
}
|
||||
@@ -2696,13 +2803,6 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#[cfg(test)]
|
||||
let timeout_duration = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
@@ -2751,18 +2851,6 @@ impl LocalDiskWrapper {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.list_multipart_index_roots(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
// The local implementation shares this binary's capability. Availability
|
||||
// and disk identity remain mutation checks and write-quorum inputs.
|
||||
@@ -2816,62 +2904,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
self.track_disk_health_with_op(
|
||||
"discover_delete_transactions",
|
||||
|| async {
|
||||
self.disk
|
||||
.discover_delete_transactions_bounded(volume, object, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
self.track_disk_health_with_op(
|
||||
"inspect_delete_transaction",
|
||||
|| async {
|
||||
self.disk
|
||||
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
self.disk
|
||||
.settle_delete_transaction_bounded(volume, request, namespace_owner)
|
||||
.await
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -3840,11 +3872,7 @@ mod tests {
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(
|
||||
!logs.contents().contains("disk_operation_slow"),
|
||||
"normal disk latency should not warn: {}",
|
||||
logs.contents()
|
||||
);
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
@@ -5718,6 +5746,115 @@ mod tests {
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
disk.make_volume("bucket").await.expect("bucket volume");
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
|
||||
disk.format_info.write().await.last_check = None;
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
disk.install_format_probe_hook(false, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(30));
|
||||
});
|
||||
let publisher = Arc::clone(&wrapper);
|
||||
let rename = tokio::spawn(async move {
|
||||
publisher
|
||||
.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&FileInfo::default(),
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), entered_rx)
|
||||
.await
|
||||
.expect("disk identity preflight starts")
|
||||
.expect("preflight hook");
|
||||
rename.abort();
|
||||
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
|
||||
let key = disk.get_object_path("bucket", "object").expect("object path");
|
||||
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
|
||||
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
|
||||
release_tx.send(()).expect("release disk identity preflight");
|
||||
if !released_before_io {
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), successor)
|
||||
.await
|
||||
.expect("preflight finishes"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
released_before_io,
|
||||
"a cancelled request must not retain the reservation before physical mutation starts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
let held = wrapper
|
||||
.io_admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("background permit");
|
||||
let fi = FileInfo::default();
|
||||
let rename = crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
wrapper.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&fi,
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
);
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
|
||||
drop(held);
|
||||
let observation = result.expect("reserved mutation must not wait while holding the object lease");
|
||||
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
|
||||
assert!(observation.rejected_before_publication());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
@@ -6181,67 +6318,6 @@ mod tests {
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
|
||||
let directories = [
|
||||
tempfile::tempdir().expect("faulted drive"),
|
||||
tempfile::tempdir().expect("healthy drive"),
|
||||
];
|
||||
let mut wrappers = Vec::new();
|
||||
for directory in &directories {
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
|
||||
wrappers.push(LocalDiskWrapper::new(disk, false));
|
||||
}
|
||||
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::clone(&wrappers[0].disk),
|
||||
false,
|
||||
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
|
||||
);
|
||||
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
|
||||
tokio::time::pause();
|
||||
let operation = || async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(())
|
||||
};
|
||||
let (faulted, healthy) = tokio::join!(
|
||||
reconnected.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
wrappers[1].track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
);
|
||||
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
|
||||
healthy.expect("healthy drive keeps its caller's deadline");
|
||||
let shorter_started = tokio::time::Instant::now();
|
||||
let shorter = wrappers[0]
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_millis(500),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
|
||||
assert!(shorter_started.elapsed() < Duration::from_secs(1));
|
||||
wrappers[0]
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
|
||||
.await
|
||||
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
|
||||
drop(scope);
|
||||
reconnected
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
|
||||
.await
|
||||
.expect("dropping the scope restores reconnected callers' deadlines");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ignored_timeout_does_not_mark_drive_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -172,8 +172,6 @@ pub enum DiskError {
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("foreground delete recovery work limit reached")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
@@ -772,7 +770,6 @@ impl Clone for DiskError {
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -832,7 +829,6 @@ impl DiskError {
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
DiskError::DeleteRecoveryWorkLimit => 0x2E,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -883,7 +879,6 @@ impl DiskError {
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1667,7 +1662,6 @@ mod tests {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
|
||||
@@ -5,7 +5,6 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -53,11 +52,7 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -130,7 +125,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
"new entries must survive the earlier volume predicate"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -169,7 +164,6 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
@@ -183,459 +177,3 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
assert_eq!(fresh.len, 11);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
|
||||
let (_directory, file) = fixture();
|
||||
let mut cache = FdCache::new();
|
||||
|
||||
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
|
||||
cache.invalidate_under("v", "missing/prefix");
|
||||
cache.invalidate_volume("v");
|
||||
cache.invalidate_exact("v", "missing/prefix/part.1").await;
|
||||
cache.clear();
|
||||
assert!(
|
||||
cache.maintenance_task.get().is_none(),
|
||||
"empty-cache invalidations must not spawn maintenance"
|
||||
);
|
||||
|
||||
// Populate a normal entry and verify matched vs unmatched filter gates directly,
|
||||
// and ensure unmatched invalidations do not enqueue pending removals.
|
||||
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(cache.tracker.live_entries(), 1);
|
||||
assert!(cache.tracker.may_match_volume("v"));
|
||||
assert!(!cache.tracker.may_match_volume("other-volume"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
|
||||
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
|
||||
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
|
||||
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
|
||||
|
||||
cache.invalidate_volume("other-volume");
|
||||
cache.invalidate_under("v", "obj-2/0198a3f0");
|
||||
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
|
||||
assert!(
|
||||
!cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"unmatched invalidations must not enqueue maintenance removals"
|
||||
);
|
||||
assert!(cache.get(&cached_key).await.is_some());
|
||||
|
||||
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
|
||||
// to per-volume unindexed_counts without degrading other volumes.
|
||||
let other_vol_key = key("v2", "shallow/part.1", false);
|
||||
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
|
||||
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
|
||||
let deep_key = key("v", deep_path, false);
|
||||
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
|
||||
"deep key in volume v must not disable prefix filtering in volume v2"
|
||||
);
|
||||
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
|
||||
assert!(cache.get(&deep_key).await.is_none());
|
||||
cache.invalidate_volume("v2");
|
||||
assert!(cache.get(&other_vol_key).await.is_none());
|
||||
|
||||
// Replacing an invalidated key before maintenance drains pending_invalidations
|
||||
// must release the old descriptor immediately and keep the replacement live
|
||||
// even when a strong reference to the old CachedFdEntry shell survives until
|
||||
// maintenance runs its Arc::ptr_eq check.
|
||||
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
|
||||
let old_weak = Arc::downgrade(&old_file);
|
||||
cache.insert(cached_key.clone(), old_file).await;
|
||||
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
|
||||
cache.invalidate_under("v", "obj-1/0198a3f0");
|
||||
assert!(cache.get(&cached_key).await.is_none());
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
old_weak.upgrade().is_none(),
|
||||
"replacing an entry must retire and drop the previous descriptor immediately"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("maintenance should drain pending invalidations");
|
||||
drop(held_old_shell);
|
||||
assert!(
|
||||
cache.get(&cached_key).await.is_some(),
|
||||
"stale pending invalidation must not evict a newer replacement"
|
||||
);
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
0
|
||||
);
|
||||
|
||||
// Dropping a cache while a live non-invalidated entry is present (even if a
|
||||
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
|
||||
// must retire the descriptor and release FdCacheTracker without cycles.
|
||||
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
|
||||
let live_drop_weak = Arc::downgrade(&live_drop_file);
|
||||
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
|
||||
let shutdown_weak = Arc::downgrade(&shutdown_file);
|
||||
let tracker_weak = Arc::downgrade(&cache.tracker);
|
||||
let cloned_moka = cache.cache.clone();
|
||||
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
|
||||
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
|
||||
cache.invalidate_under("v", "shutdown");
|
||||
if let Some(handle) = cache.maintenance_task.take() {
|
||||
handle.abort();
|
||||
let _ = handle.await;
|
||||
}
|
||||
drop(cache);
|
||||
assert!(
|
||||
live_drop_weak.upgrade().is_none(),
|
||||
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
|
||||
);
|
||||
assert!(
|
||||
shutdown_weak.upgrade().is_none(),
|
||||
"dropping cache when maintenance is aborted must still release descriptors"
|
||||
);
|
||||
assert!(
|
||||
tracker_weak.upgrade().is_none(),
|
||||
"dropping cache must release FdCacheTracker without reference cycles"
|
||||
);
|
||||
drop(cloned_moka);
|
||||
|
||||
// Short-TTL expired entries (including mixed expired and fresh entries) must
|
||||
// be retired by invalidate_under and clear() even though Moka's iterator skips
|
||||
// expired entries.
|
||||
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
|
||||
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
|
||||
let expired_weak = Arc::downgrade(&expired_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
|
||||
let fresh_weak = Arc::downgrade(&fresh_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
|
||||
ttl_cache.invalidate_under("v", "ttl-obj");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("invalidate_under must retire both expired and fresh matching descriptors");
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
|
||||
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
|
||||
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
|
||||
ttl_cache
|
||||
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
|
||||
.await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
|
||||
let clear_live_weak = Arc::downgrade(&clear_live_file);
|
||||
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
|
||||
ttl_cache.clear();
|
||||
assert!(
|
||||
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
|
||||
"clear() must synchronously retire both TTL-expired and live descriptors"
|
||||
);
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
|
||||
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
|
||||
|
||||
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
|
||||
let paths = [
|
||||
"a/b",
|
||||
"a/b/part.1",
|
||||
"a/b/dir/part.2",
|
||||
"a/bc",
|
||||
"a/bc/part.1",
|
||||
"prefix/obj-0001.bin/0198a3f0/part.1",
|
||||
"prefix/obj-0002.bin/0198a3f0/part.1",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
|
||||
];
|
||||
let prefixes = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/",
|
||||
"a/b/dir",
|
||||
"a/bc",
|
||||
"prefix",
|
||||
"prefix/obj-0001.bin",
|
||||
"prefix/obj-0001.bin/0198a3f0",
|
||||
"d1/d2/d3/d4/d5/c_missing",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
|
||||
"nonexistent/prefix",
|
||||
];
|
||||
|
||||
let mut candidate_keys = Vec::new();
|
||||
for v in volumes {
|
||||
for p in paths {
|
||||
for direct in [false, true] {
|
||||
candidate_keys.push(key(v, p, direct));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut next_u64 = || {
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 7;
|
||||
rng ^= rng << 17;
|
||||
rng
|
||||
};
|
||||
|
||||
for step in 0..1000 {
|
||||
let op = next_u64() % 100;
|
||||
if op < 45 {
|
||||
// Insert or replace (either fresh or stale generation)
|
||||
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: step,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let cur_gen = cache.generation();
|
||||
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
|
||||
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
|
||||
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
|
||||
if !stale {
|
||||
oracle.insert(k, entry);
|
||||
}
|
||||
} else if op < 65 {
|
||||
// Exact invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let p = paths[(next_u64() as usize) % paths.len()];
|
||||
cache.invalidate_exact(v, p).await;
|
||||
oracle.remove(&key(v, p, false));
|
||||
oracle.remove(&key(v, p, true));
|
||||
} else if op < 88 {
|
||||
// Prefix invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
|
||||
cache.invalidate_under(v, raw_prefix);
|
||||
let trimmed = raw_prefix.trim_end_matches('/');
|
||||
oracle.retain(|k, _| {
|
||||
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
|
||||
});
|
||||
} else if op < 97 {
|
||||
// Volume invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
cache.invalidate_volume(v);
|
||||
oracle.retain(|k, _| k.volume != v);
|
||||
} else {
|
||||
// Full clear
|
||||
cache.clear();
|
||||
oracle.clear();
|
||||
}
|
||||
|
||||
// Verify lockstep equivalence against the reference oracle after every operation.
|
||||
for k in &candidate_keys {
|
||||
let got = cache.get(k).await;
|
||||
let expected = oracle.get(k);
|
||||
match (got, expected) {
|
||||
(Some(g), Some(e)) => assert!(
|
||||
Arc::ptr_eq(&g, e),
|
||||
"step {step}: pointer mismatch for key ({}, {}, {})",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct
|
||||
),
|
||||
(None, None) => {}
|
||||
(g, e) => panic!(
|
||||
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct,
|
||||
g.is_some(),
|
||||
e.is_some()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear must drain all live entries and pending invalidations");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = Arc::new(FdCache::new());
|
||||
|
||||
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
|
||||
// at `entry_shards[shard].lock()` after it has passed the initial generation
|
||||
// check and incremented `volume_counts`, bump generation via an unmatched
|
||||
// invalidation, and verify the post-`new()` recheck refuses publication and
|
||||
// preserves the existing entry.
|
||||
let seam_key = key("v-seam", "obj-1/part.1", false);
|
||||
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
|
||||
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
|
||||
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
|
||||
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
|
||||
let stale_gen = cache.generation();
|
||||
let stale_entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: 999_999,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let blocked_insert = {
|
||||
let hold = cache.tracker.entry_shards[entry_shard].lock();
|
||||
let cache_clone = Arc::clone(&cache);
|
||||
let seam_key_clone = seam_key.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
|
||||
});
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"insert never reached CachedFdEntry::new with the previous entry still live"
|
||||
);
|
||||
std::thread::yield_now();
|
||||
}
|
||||
cache.invalidate_under("v-seam", "unmatched-prefix");
|
||||
drop(hold);
|
||||
task
|
||||
};
|
||||
blocked_insert.await.expect("blocked insert task should complete");
|
||||
let preserved = cache
|
||||
.get(&seam_key)
|
||||
.await
|
||||
.expect("refused stale insert must not retire a valid previous entry");
|
||||
assert!(Arc::ptr_eq(&preserved, &initial_entry));
|
||||
cache.invalidate_volume("v-seam");
|
||||
|
||||
let floor_v1 = Arc::new(AtomicU64::new(0));
|
||||
let floor_v2 = Arc::new(AtomicU64::new(0));
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for t in 0..8usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let file = Arc::clone(&file);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
let k = key(vol, &path, i.is_multiple_of(2));
|
||||
let g = cache.generation();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: g,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
cache.insert_if_fresh(k.clone(), entry, g).await;
|
||||
let min_gen = if is_v1 {
|
||||
floor_v1.load(Ordering::SeqCst)
|
||||
} else {
|
||||
floor_v2.load(Ordering::SeqCst)
|
||||
};
|
||||
if let Some(hit) = cache.get(&k).await {
|
||||
assert!(
|
||||
hit.len >= min_gen,
|
||||
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
|
||||
hit.len
|
||||
);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for t in 0..4usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
match i % 4 {
|
||||
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
|
||||
1 => {
|
||||
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
cache.invalidate_exact(vol, &p).await;
|
||||
}
|
||||
2 => {
|
||||
let before = cache.generation();
|
||||
cache.invalidate_volume(vol);
|
||||
let floor = before + 1;
|
||||
if is_v1 {
|
||||
floor_v1.fetch_max(floor, Ordering::SeqCst);
|
||||
} else {
|
||||
floor_v2.fetch_max(floor, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.await.expect("worker task should succeed");
|
||||
}
|
||||
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear after concurrent stress must drain all live entries");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
+1494
-1361
File diff suppressed because it is too large
Load Diff
@@ -444,12 +444,29 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
impl LocalRenamePreflightRejection {
|
||||
pub(in crate::disk) fn not_dispatched() -> Self {
|
||||
Self(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RenameDataState {
|
||||
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
|
||||
Self {
|
||||
metadata_reservation: Some(reservation),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep optional native MPU state machines out of ordinary rename frames.
|
||||
// Construct the future here as well as pinning it, so its temporary stays off
|
||||
// the caller's poll stack. Call only from the existing native MPU branches.
|
||||
@@ -486,14 +503,20 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// Pre-acquired metadata must never reverse quota or multipart-source
|
||||
// lock order. These paths need their own reservation protocol.
|
||||
if state.metadata_reservation.is_some()
|
||||
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
|
||||
{
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
|
||||
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
@@ -589,7 +612,27 @@ impl LocalDisk {
|
||||
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
|
||||
None => state.namespace_owner.take(),
|
||||
};
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
|
||||
// A reserved lease is single-use and bound to the exact disk path.
|
||||
// Keep the independent namespace/source owner alongside it; a pre-held
|
||||
// mutex cannot be acquired again without deadlocking.
|
||||
let reservation_fence = state
|
||||
.metadata_reservation
|
||||
.as_ref()
|
||||
.and_then(|reservation| reservation.fence());
|
||||
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
|
||||
if multipart_staging.is_some() || quota_fence_claim.is_some() {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let lease = reservation
|
||||
.into_lease_for(&root_destination_object_path, self.reservation_instance)
|
||||
.inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
Arc::new((lease, metadata_owner))
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
|
||||
};
|
||||
drop(metadata_wait);
|
||||
#[cfg(unix)]
|
||||
let parent_receipt = if native_timing {
|
||||
@@ -603,17 +646,23 @@ impl LocalDisk {
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let mut receipt_mutation = native_timing
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
|
||||
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&root_destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
Some(metadata_owner),
|
||||
)
|
||||
.await;
|
||||
drop(namespace_wait);
|
||||
// The volume read lease prevents deletion of this disk's bucket until
|
||||
// publication finishes. Recheck after its wait, before any mutation.
|
||||
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -2552,8 +2601,23 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
|
||||
) -> super::super::RenameDataObservation {
|
||||
let (namespace_owner, metadata_reservation) = owners;
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
..Default::default()
|
||||
};
|
||||
let result = self
|
||||
|
||||
@@ -5,28 +5,6 @@ use crate::disk::{
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(super) struct RecoveryOperationScope {
|
||||
pub(super) budget: Arc<crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
may_publish_receipt: bool,
|
||||
receipt_published: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static RECOVERY_OPERATION: Arc<RecoveryOperationScope>;
|
||||
}
|
||||
|
||||
pub(super) fn recovery_operation_scope() -> Option<Arc<RecoveryOperationScope>> {
|
||||
RECOVERY_OPERATION.try_with(Arc::clone).ok()
|
||||
}
|
||||
|
||||
impl RecoveryOperationScope {
|
||||
pub(super) fn receipt_published(&self) {
|
||||
if self.may_publish_receipt {
|
||||
self.receipt_published.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const PREPARED: &str = "delete-prepared";
|
||||
pub(super) const TARGET: &str = "delete-target";
|
||||
pub(super) const FINISHED: &str = "delete-commit-finished";
|
||||
@@ -41,35 +19,24 @@ struct DiscoverySession {
|
||||
touched: Instant,
|
||||
}
|
||||
|
||||
/// Plain (non-transaction) children of an object-scoped discovery root cost one
|
||||
/// budget entry per this many names: enough to bound a hostile prefix without
|
||||
/// making a real folder prefix undeletable.
|
||||
const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
|
||||
|
||||
impl DiscoverySession {
|
||||
/// Consume one bounded step, returning ownership only when traversal remains.
|
||||
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
|
||||
.map(|(transactions, next, _)| (transactions, next))
|
||||
}
|
||||
|
||||
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
fn advance(mut self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
if self.frames.is_empty() {
|
||||
match std::fs::read_dir(&self.root) {
|
||||
Ok(entries) => self.frames.push((String::new(), entries)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
return Ok((Vec::new(), None, 0));
|
||||
return Ok((Vec::new(), None));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut seen_candidates = HashSet::new();
|
||||
let mut visited = 0u32;
|
||||
let mut plain_children = 0u32;
|
||||
let mut visited = 0usize;
|
||||
while !self.frames.is_empty()
|
||||
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
|
||||
&& visited < max_entries
|
||||
&& visited < 4096
|
||||
{
|
||||
let (parent, entries) = self
|
||||
.frames
|
||||
@@ -82,6 +49,7 @@ impl DiscoverySession {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
visited += 1;
|
||||
let name = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
@@ -89,19 +57,6 @@ impl DiscoverySession {
|
||||
if !safe_path(&name, false) || name.contains('/') {
|
||||
return Err(std::io::Error::from(ErrorKind::InvalidData));
|
||||
}
|
||||
// An object-scoped scan only ever descends into transaction (UUID)
|
||||
// directories at the root. Plain child keys of a prefix directory are
|
||||
// skipped by name without a stat; they are charged at one entry per
|
||||
// readdir batch so the work stays bounded while deleting a missing key
|
||||
// that names a prefix with thousands of children can still complete.
|
||||
if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
|
||||
plain_children += 1;
|
||||
if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
|
||||
visited += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
visited += 1;
|
||||
let relative = if parent.is_empty() {
|
||||
name.clone()
|
||||
} else {
|
||||
@@ -201,7 +156,7 @@ impl DiscoverySession {
|
||||
.map(|(object, transaction)| DeleteTransactionId { object, transaction })
|
||||
.collect();
|
||||
let next = (!self.frames.is_empty()).then_some(self);
|
||||
Ok((transactions, next, visited))
|
||||
Ok((transactions, next))
|
||||
}
|
||||
|
||||
async fn has_evidence(mut self, disk: &LocalDisk, path: &Path) -> Result<bool> {
|
||||
@@ -625,41 +580,12 @@ fn explicit_delete_referenced_data_dirs(
|
||||
Ok(Some(referenced))
|
||||
}
|
||||
|
||||
pub(super) async fn next_recovery_entry(entries: &mut tokio::fs::ReadDir) -> std::io::Result<Option<tokio::fs::DirEntry>> {
|
||||
if let Some(scope) = recovery_operation_scope() {
|
||||
scope.budget.reserve(1).map_err(std::io::Error::from)?;
|
||||
}
|
||||
entries.next_entry().await
|
||||
}
|
||||
|
||||
pub(super) fn recovery_entries(
|
||||
mut entries: std::fs::ReadDir,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> impl Iterator<Item = std::io::Result<std::fs::DirEntry>> + '_ {
|
||||
let mut finished = false;
|
||||
std::iter::from_fn(move || {
|
||||
if finished {
|
||||
return None;
|
||||
}
|
||||
if let Some(budget) = budget
|
||||
&& let Err(error) = budget.reserve(1)
|
||||
{
|
||||
finished = true;
|
||||
return Some(Err(error.into()));
|
||||
}
|
||||
let entry = entries.next();
|
||||
finished = entry.is_none();
|
||||
entry
|
||||
})
|
||||
}
|
||||
|
||||
fn explicit_delete_plan_bounded(
|
||||
fn explicit_delete_plan(
|
||||
root: &Path,
|
||||
transaction: Uuid,
|
||||
original: Option<&FileMeta>,
|
||||
journal: Option<&DeleteRecoveryJournal>,
|
||||
current_missing: bool,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Result<Option<ExplicitDeletePlan>> {
|
||||
let Some(referenced) = explicit_delete_referenced_data_dirs(original, journal, current_missing)? else {
|
||||
return Ok(None);
|
||||
@@ -676,7 +602,7 @@ fn explicit_delete_plan_bounded(
|
||||
let mut candidates = Vec::new();
|
||||
match std::fs::read_dir(&rollback) {
|
||||
Ok(entries) => {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let kind = entry.file_type().map_err(to_file_error)?;
|
||||
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
@@ -708,7 +634,7 @@ fn explicit_delete_plan_bounded(
|
||||
}
|
||||
match std::fs::read_dir(root) {
|
||||
Ok(entries) => {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil()) else {
|
||||
@@ -732,7 +658,7 @@ fn explicit_delete_plan_bounded(
|
||||
candidates.dedup();
|
||||
for dir in candidates {
|
||||
let mut has_parts = false;
|
||||
for entry in recovery_entries(std::fs::read_dir(&dir).map_err(to_file_error)?, budget) {
|
||||
for entry in std::fs::read_dir(&dir).map_err(to_file_error)? {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
if !entry.file_type().map_err(to_file_error)?.is_file() {
|
||||
return Ok(None);
|
||||
@@ -771,27 +697,15 @@ fn canonical_uuid(name: &str) -> Option<Uuid> {
|
||||
Uuid::parse_str(name).ok().filter(|id| name == id.hyphenated().to_string())
|
||||
}
|
||||
|
||||
pub(super) fn missing_marker_preserves_child_prefix(journal: &DeleteRecoveryJournal) -> bool {
|
||||
journal.original_digest.is_none()
|
||||
&& journal.original_versions.is_empty()
|
||||
&& journal.data_dirs.is_empty()
|
||||
&& journal.target_digest.is_some()
|
||||
&& !journal.requests.is_empty()
|
||||
&& journal
|
||||
.requests
|
||||
.iter()
|
||||
.all(|request| request.deleted && request.version_id.is_some_and(|id| !id.is_nil()))
|
||||
}
|
||||
|
||||
pub(super) fn inspection_with_markers_bounded(
|
||||
pub(super) fn inspection_with_markers(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Result<(DeleteRecoveryInspection, bool)> {
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?;
|
||||
let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?;
|
||||
let prepared = regular_bytes(&rollback.join(PREPARED))?;
|
||||
let target = regular_metadata_bytes(&rollback.join(TARGET))?;
|
||||
let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
@@ -825,20 +739,6 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// A failed creation of a marker over a child named `xl.meta` did not
|
||||
// replace that directory. Older releases nevertheless left a journal.
|
||||
// Only this bound, originally absent generation may treat it as absence.
|
||||
let current_path = root.join(STORAGE_FORMAT_FILE);
|
||||
let current = if !committed
|
||||
&& before.is_none()
|
||||
&& (journal.as_ref().is_some_and(missing_marker_preserves_child_prefix)
|
||||
|| (journal.is_none() && target.is_none() && receipt_bytes.is_none()))
|
||||
&& std::fs::symlink_metadata(¤t_path).is_ok_and(|metadata| metadata.is_dir())
|
||||
{
|
||||
None
|
||||
} else {
|
||||
regular_metadata_bytes(¤t_path)?
|
||||
};
|
||||
let backup_digest = before.as_deref().map(digest);
|
||||
let current_digest = current.as_deref().map(digest);
|
||||
let prepared_digest = prepared.as_deref().map(digest);
|
||||
@@ -881,7 +781,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
};
|
||||
if let Some(entries) = entries {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let kind = entry.file_type().map_err(to_file_error)?;
|
||||
if !kind.is_file() && !kind.is_dir() {
|
||||
@@ -898,7 +798,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len()));
|
||||
if kind.is_dir() && id.is_some_and(|id| !id.is_nil()) {
|
||||
has_other_uuid_dir |= id != Some(transaction);
|
||||
for child in recovery_entries(std::fs::read_dir(entry.path()).map_err(to_file_error)?, budget) {
|
||||
for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? {
|
||||
let child = child.map_err(to_file_error)?;
|
||||
let kind = child.file_type().map_err(to_file_error)?;
|
||||
if !kind.is_dir() && !kind.is_file() {
|
||||
@@ -944,7 +844,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
}
|
||||
if name == transaction.to_string() && kind.is_dir() {
|
||||
has_staged_dir = true;
|
||||
for file in recovery_entries(std::fs::read_dir(child.path()).map_err(to_file_error)?, budget) {
|
||||
for file in std::fs::read_dir(child.path()).map_err(to_file_error)? {
|
||||
let file = file.map_err(to_file_error)?;
|
||||
let kind = file.file_type().map_err(to_file_error)?;
|
||||
let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
@@ -1068,8 +968,7 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
.is_some_and(|referenced| !referenced.contains(&transaction))
|
||||
}
|
||||
DeleteRecoveryState::LegacyAmbiguous | DeleteRecoveryState::NewPrepared => {
|
||||
explicit_delete_plan_bounded(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none(), budget)?
|
||||
.is_some()
|
||||
explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some()
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
@@ -1103,17 +1002,21 @@ pub(super) fn inspection_with_markers_bounded(
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
|
||||
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.remove(rollback)
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
@@ -1140,7 +1043,6 @@ impl LocalDisk {
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
|
||||
let recovery_scope = recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
Ok((|| {
|
||||
let entries = match std::fs::read_dir(&root) {
|
||||
@@ -1150,7 +1052,7 @@ impl LocalDisk {
|
||||
};
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut visited = 0usize;
|
||||
for entry in recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())) {
|
||||
for entry in entries {
|
||||
visited += 1;
|
||||
if visited > 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
@@ -1303,42 +1205,6 @@ impl LocalDisk {
|
||||
.map_err(|error| to_file_error(error).into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn public_delete_discovery_sessions_for_test(&self) -> usize {
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.filter(|session| session.disk_root == self.io_root())
|
||||
.count()
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_discover_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<super::super::BoundedDeleteRecoveryDiscovery> {
|
||||
if !(1..=super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES).contains(&max_entries) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let path = self.delete_discovery_path(volume, Some(object))?;
|
||||
let session = self.new_delete_discovery_session(volume, Some(object), &path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
|
||||
self.validate_delete_discovery_owner(&session)?;
|
||||
// The blocking owner drops every directory handle even if the RPC waiter disappears.
|
||||
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
|
||||
Ok(super::super::BoundedDeleteRecoveryDiscovery {
|
||||
transactions: if next.is_none() { transactions } else { Vec::new() },
|
||||
visited,
|
||||
complete: next.is_none(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| to_file_error(error).into())
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_discover(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1386,39 +1252,6 @@ impl LocalDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_inspect_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<super::super::BoundedDeleteRecoveryInspection> {
|
||||
if !safe_path(object, false)
|
||||
|| transaction.is_nil()
|
||||
|| !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&max_entries)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let root = self.get_object_path(volume, object)?;
|
||||
let owner = os::acquire_metadata_mutation_lease(&root, None).await;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
|
||||
Ok(
|
||||
inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
|
||||
super::super::BoundedDeleteRecoveryInspection {
|
||||
inspection,
|
||||
visited: max_entries - budget.remaining(),
|
||||
}
|
||||
}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_inspect(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1458,22 +1291,9 @@ impl LocalDisk {
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
|
||||
let scope = recovery_operation_scope();
|
||||
// Journal inspections ride the durable-mutation lane (the fused delete
|
||||
// prepare already does): a bulk delete publishes one transaction per
|
||||
// key, and the one-worker foreground namespace lane would serialize
|
||||
// those per-drive publishes behind every other mutation.
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
Ok(inspection_with_markers_bounded(
|
||||
&root,
|
||||
&volume,
|
||||
&object,
|
||||
transaction,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
))
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
async fn activate_delete_preparation(
|
||||
@@ -1671,49 +1491,6 @@ impl LocalDisk {
|
||||
self.activate_delete_preparation(&journal, original, target, owner).await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_settle_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: super::super::BoundedDeleteRecoverySettlement,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<super::super::BoundedDeleteRecoverySettlementReply> {
|
||||
if !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&request.max_entries) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let scope = Arc::new(RecoveryOperationScope {
|
||||
budget: Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(request.max_entries)),
|
||||
may_publish_receipt: matches!(
|
||||
&request.action,
|
||||
DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. }
|
||||
),
|
||||
receipt_published: std::sync::atomic::AtomicBool::new(false),
|
||||
});
|
||||
let result = RECOVERY_OPERATION
|
||||
.scope(
|
||||
Arc::clone(&scope),
|
||||
self.delete_recovery_settle(
|
||||
volume,
|
||||
&request.object,
|
||||
request.transaction,
|
||||
request.expected_fingerprint,
|
||||
request.action,
|
||||
owner,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let outcome = match result {
|
||||
Ok(inspection) => super::super::DeleteRecoverySettlementOutcome::Inspected(Box::new(inspection)),
|
||||
Err(DiskError::DeleteRecoveryWorkLimit) if scope.receipt_published.load(std::sync::atomic::Ordering::Acquire) => {
|
||||
super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
Ok(super::super::BoundedDeleteRecoverySettlementReply {
|
||||
outcome,
|
||||
visited: request.max_entries - scope.budget.remaining(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_recovery_settle(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1912,26 +1689,14 @@ impl LocalDisk {
|
||||
let plan_root = root.clone();
|
||||
let missing = observed.current_digest.is_none();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
|
||||
let scope = recovery_operation_scope();
|
||||
// Same durable-mutation lane as the journal inspection above: this
|
||||
// plan closure runs inside every settle on the bulk-delete publish
|
||||
// path.
|
||||
let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
let plan = os::run_blocking_namespace_operation(lease, move || {
|
||||
Ok((|| {
|
||||
let before =
|
||||
regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?
|
||||
.as_deref()
|
||||
.map(FileMeta::load)
|
||||
.transpose()?;
|
||||
explicit_delete_plan_bounded(
|
||||
&plan_root,
|
||||
transaction,
|
||||
before.as_ref(),
|
||||
None,
|
||||
missing,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)?
|
||||
.ok_or(DiskError::FileCorrupt)
|
||||
explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
@@ -2196,7 +1961,6 @@ impl LocalDisk {
|
||||
let (rollback_buf, root_buf, volume_str, object_str) =
|
||||
(rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
|
||||
let publication_root = self.publication_root.clone();
|
||||
let scope = recovery_operation_scope();
|
||||
return os::run_blocking_namespace_operation_with_sync(receipt_lease, sync, move || {
|
||||
let _publication_root = publication_root;
|
||||
for (path, _) in &file_leases {
|
||||
@@ -2229,14 +1993,8 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
let inspected = inspection_with_markers_bounded(
|
||||
&root_buf,
|
||||
&volume_str,
|
||||
&object_str,
|
||||
transaction,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)
|
||||
.map(|(state, _)| state);
|
||||
let inspected =
|
||||
inspection_with_markers(&root_buf, &volume_str, &object_str, transaction).map(|(state, _)| state);
|
||||
drop((file_leases, prune_dirs));
|
||||
Ok(inspected)
|
||||
})
|
||||
@@ -2393,7 +2151,6 @@ impl LocalDisk {
|
||||
let volume_str = volume.to_owned();
|
||||
let object_str = object.to_owned();
|
||||
|
||||
let recovery_scope = recovery_operation_scope();
|
||||
let res = os::run_blocking_namespace_operation_with_sync(
|
||||
lease,
|
||||
sync,
|
||||
@@ -2440,9 +2197,7 @@ impl LocalDisk {
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in
|
||||
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
|
||||
{
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
@@ -2655,25 +2410,12 @@ impl LocalDisk {
|
||||
Ok(Some(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local_with_budget(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
@@ -2752,7 +2494,7 @@ impl LocalDisk {
|
||||
if sole_transaction {
|
||||
match std::fs::read_dir(&root) {
|
||||
Ok(entries) => {
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let Ok(name) = entry.file_name().into_string() else {
|
||||
sole_transaction = false;
|
||||
@@ -2800,7 +2542,6 @@ impl LocalDisk {
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2808,26 +2549,6 @@ impl LocalDisk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
self.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match res {
|
||||
@@ -2894,7 +2615,7 @@ impl LocalDisk {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
@@ -4398,100 +4119,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_marker_collision_rejects_new_journals_and_recovers_legacy_proof() {
|
||||
let (temp, disk, _info, original) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
let staged_child = temp.path().join("child-fixture");
|
||||
fs::rename(&root, &staged_child).await.expect("move intact object");
|
||||
fs::create_dir(&root).await.expect("create parent prefix");
|
||||
fs::rename(staged_child, root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("seed an independent object named object/xl.meta");
|
||||
let transaction = Uuid::new_v4();
|
||||
let marker = FileInfo {
|
||||
volume: "recovery".to_owned(),
|
||||
name: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
disk.delete_version_recoverable(
|
||||
"recovery",
|
||||
"object",
|
||||
marker.clone(),
|
||||
true,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a child xl.meta directory cannot be replaced by a marker");
|
||||
assert!(!root.join(transaction.to_string()).exists(), "rejection must precede journal publication");
|
||||
|
||||
// Reproduce the durable evidence written by releases that prepared a
|
||||
// missing-object marker before attempting to replace the directory.
|
||||
disk.prepare_delete_recovery(
|
||||
"recovery",
|
||||
"object",
|
||||
None,
|
||||
&[marker],
|
||||
None,
|
||||
DeletePreparationContext {
|
||||
transaction,
|
||||
bucket_incarnation: None,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed legacy prepared marker");
|
||||
let journal_path = root.join(transaction.to_string()).join(PREPARED);
|
||||
let journal_bytes = fs::read(&journal_path).await.expect("prepared journal");
|
||||
let mut incompatible: DeleteRecoveryJournal = serde_json::from_slice(&journal_bytes).expect("journal");
|
||||
incompatible.original_digest = Some([7; 32]);
|
||||
fs::write(&journal_path, serde_json::to_vec(&incompatible).expect("serialize journal"))
|
||||
.await
|
||||
.expect("write incompatible evidence");
|
||||
assert!(
|
||||
disk.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.is_err(),
|
||||
"a journal that claims existing metadata cannot ignore a directory collision"
|
||||
);
|
||||
fs::write(&journal_path, journal_bytes)
|
||||
.await
|
||||
.expect("restore exact legacy proof");
|
||||
let observed = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("inspect legacy proof");
|
||||
assert_eq!(observed.state, DeleteRecoveryState::NewPrepared);
|
||||
let recovered = disk
|
||||
.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback)
|
||||
.await
|
||||
.expect("rollback legacy marker without unlinking the child directory");
|
||||
assert_eq!(recovered.state, DeleteRecoveryState::Absent);
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
assert_eq!(
|
||||
disk.read_all("recovery", "object/xl.meta/xl.meta")
|
||||
.await
|
||||
.expect("child metadata")
|
||||
.as_ref(),
|
||||
original
|
||||
);
|
||||
assert!(
|
||||
disk.read_version("", "recovery", "object/xl.meta", "", &Default::default())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
discover_all_keys(&disk, Some("object")).await.is_empty(),
|
||||
"no proof may keep replacement heal pending"
|
||||
);
|
||||
}
|
||||
|
||||
// Data movement copies a delete marker onto a target that holds no
|
||||
// metadata yet. Rolling back that sub-quorum write removes the marker it
|
||||
// published, and must not report the backup it never wrote as missing:
|
||||
@@ -6135,152 +5762,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_inspection_charges_root_and_uuid_children() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let full = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
|
||||
.await
|
||||
.expect("full bounded inspection");
|
||||
assert!(full.visited > 1, "both root and transaction directory entries are charged");
|
||||
let error = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, full.visited - 1)
|
||||
.await
|
||||
.expect_err("nested entry exceeds allowance");
|
||||
assert_eq!(error, DiskError::DeleteRecoveryWorkLimit);
|
||||
let unchanged = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("unchanged proof");
|
||||
assert_eq!(full.inspection.fingerprint, unchanged.fingerprint);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_receipt_publication_distinguishes_deferred_inspection_from_write_failure() {
|
||||
for write_fails in [false, true] {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let full = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
|
||||
.await
|
||||
.expect("measure preparation work");
|
||||
let receipt = disk
|
||||
.io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}"))
|
||||
.expect("receipt path");
|
||||
if write_fails {
|
||||
let destination = receipt.clone();
|
||||
set_owned_file_before_open(&receipt, move || std::fs::create_dir(&destination).expect("reject receipt rename"));
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
"recovery",
|
||||
super::super::super::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: full.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: full.visited,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if write_fails {
|
||||
assert!(result.is_err(), "a failed receipt write never counts as publication");
|
||||
assert!(receipt.is_dir());
|
||||
} else {
|
||||
let result = result.expect("persisted receipt is acknowledged independently of post-write inspection");
|
||||
assert!(matches!(
|
||||
result.outcome,
|
||||
super::super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
|
||||
));
|
||||
assert_eq!(result.visited, full.visited);
|
||||
let state = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("durable receipt");
|
||||
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
fs::create_dir_all(&path).await.expect("physical prefix");
|
||||
// Transaction-shaped children are the entries a bounded probe pays for.
|
||||
for _ in 0..64 {
|
||||
fs::create_dir(path.join(Uuid::new_v4().to_string()))
|
||||
.await
|
||||
.expect("legitimate child");
|
||||
}
|
||||
for _ in 0..140 {
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 8)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
assert_eq!(page.visited, 8);
|
||||
assert!(!page.complete, "an empty partial page cannot establish absence");
|
||||
assert!(page.transactions.is_empty());
|
||||
}
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 128)
|
||||
.await
|
||||
.expect("complete probe");
|
||||
assert!(page.complete);
|
||||
assert_eq!(page.visited, 64);
|
||||
assert!(page.transactions.is_empty());
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let before = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("before probe");
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "object", 4096)
|
||||
.await
|
||||
.expect("complete evidence");
|
||||
assert!(page.complete);
|
||||
assert!(page.transactions.iter().any(|id| id.transaction == transaction));
|
||||
let after = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("after probe");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
assert!(
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.all(|session| session.disk_root != disk.io_root())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_cancelled_bounded_discovery_drops_private_session() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("object")).expect("probe scope");
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
|
||||
let mut probe = Box::pin(disk.discover_delete_transactions_bounded("recovery", "object", 4096));
|
||||
assert!(futures::poll!(probe.as_mut()).is_pending());
|
||||
drop(probe);
|
||||
assert!(
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.all(|session| session.disk_root != disk.io_root())
|
||||
);
|
||||
drop(lease);
|
||||
assert!(
|
||||
disk.discover_delete_transactions_bounded("recovery", "object", 4096)
|
||||
.await
|
||||
.expect("subsequent probe")
|
||||
.complete
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_evidence_probes_do_not_register_public_cursors() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,7 +117,6 @@ pub(super) struct LoadedSource {
|
||||
}
|
||||
|
||||
pub(super) struct SourceStore {
|
||||
multipart_index: Arc<super::multipart_index::IndexOwner>,
|
||||
pub disk_id: Uuid,
|
||||
pub physical_slot: usize,
|
||||
pub disk_ids: Vec<Uuid>,
|
||||
@@ -187,7 +186,6 @@ impl LocalDisk {
|
||||
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
|
||||
Ok(SourceStore {
|
||||
multipart_index: self.multipart_index.clone(),
|
||||
disk_id,
|
||||
physical_slot,
|
||||
disk_ids,
|
||||
@@ -272,17 +270,21 @@ struct ActiveMultipartSourceEntry {
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
|
||||
let source = source.as_ref();
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
@@ -302,50 +304,56 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| entry.fi.data_dir)
|
||||
.flatten()
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
@@ -354,9 +362,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
@@ -365,29 +373,34 @@ pub(super) fn finish_active_multipart_part_transaction(
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
map.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
@@ -398,11 +411,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
@@ -757,9 +770,6 @@ impl SourceStore {
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
@@ -1926,7 +1936,6 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
|
||||
+116
-182
@@ -27,7 +27,6 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub mod multipart_index;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
|
||||
@@ -107,6 +106,8 @@ pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
pub(crate) metadata_reservation_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
@@ -123,6 +124,13 @@ impl RenameDataObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn not_dispatched(error: DiskError) -> Self {
|
||||
Self {
|
||||
result: Err(error),
|
||||
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
@@ -312,6 +320,82 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Prepare an inert single-use reservation on this disk.
|
||||
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
|
||||
pub async fn prepare_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.prepare_object_metadata(bucket, object),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh an inert token while its sender stages the write.
|
||||
pub async fn renew_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.renew_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token in object mutation order.
|
||||
pub async fn activate_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.activate_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. False does not prove that a claim has
|
||||
/// not already started a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
match self {
|
||||
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
|
||||
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
@@ -415,11 +499,10 @@ impl Disk {
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(None),
|
||||
@@ -433,19 +516,11 @@ impl Disk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(false),
|
||||
}
|
||||
@@ -522,17 +597,6 @@ impl Disk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for Disk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
|
||||
@@ -576,17 +640,6 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -599,35 +652,6 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1312,18 +1336,33 @@ impl Disk {
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
Disk::Remote(remote_disk) => {
|
||||
if guards.metadata_reservation.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
if let Some(token) = guards.metadata_reservation_token {
|
||||
if guards.scanner_publication_lease_token.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
return RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1574,15 +1613,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
fn host_name(&self) -> String;
|
||||
fn endpoint(&self) -> Endpoint;
|
||||
async fn close(&self) -> Result<()>;
|
||||
/// Return candidates only while this disk maintains complete boot-local coverage.
|
||||
/// Unsupported or unready disks must never return an authoritative empty page.
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
|
||||
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
|
||||
|
||||
@@ -1760,33 +1790,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
/// Bounded foreground inventory owns all traversal state until return or cancellation.
|
||||
/// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_transaction: Uuid,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_request: BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
/// A read-only capability check; unsupported implementations must fail closed.
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Err(DiskError::GuardedDeleteUnsupported)
|
||||
@@ -2172,13 +2175,6 @@ pub struct WalkDirOptions {
|
||||
#[serde(default)]
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
|
||||
/// Bound hidden metadata read while proving a non-recursive prefix visible.
|
||||
#[serde(default)]
|
||||
pub hidden_entry_budget: usize,
|
||||
/// A prefix whose visibility was not established before the preceding cut.
|
||||
#[serde(default)]
|
||||
pub pending_prefix: Option<String>,
|
||||
|
||||
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
|
||||
pub report_notfound: bool,
|
||||
|
||||
@@ -2511,62 +2507,9 @@ pub enum DeleteRecoveryAction {
|
||||
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
|
||||
}
|
||||
|
||||
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub transactions: Vec<DeleteTransactionId>,
|
||||
pub visited: u32,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlement {
|
||||
pub object: String,
|
||||
pub transaction: Uuid,
|
||||
pub expected_fingerprint: DeleteRecoveryDigest,
|
||||
pub action: DeleteRecoveryAction,
|
||||
pub max_entries: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoverySettlementOutcome {
|
||||
Inspected(Box<DeleteRecoveryInspection>),
|
||||
ReceiptPublishedInspectionDeferred,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlementReply {
|
||||
pub outcome: DeleteRecoverySettlementOutcome,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryInspection {
|
||||
pub inspection: DeleteRecoveryInspection,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcOperation {
|
||||
SettleBounded(BoundedDeleteRecoverySettlement),
|
||||
InspectBounded {
|
||||
object: String,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
},
|
||||
DiscoverBounded {
|
||||
object: String,
|
||||
max_entries: u32,
|
||||
},
|
||||
InspectOrphanDataDir {
|
||||
path: String,
|
||||
},
|
||||
@@ -2604,11 +2547,7 @@ impl DeleteRecoveryRpcOperation {
|
||||
/// Full metadata application and adoption share the existing mutation envelope.
|
||||
pub fn max_wire_payload_bytes(&self) -> usize {
|
||||
match self {
|
||||
Self::SettleBounded(BoundedDeleteRecoverySettlement {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
..
|
||||
})
|
||||
| Self::ApplyVersion { .. }
|
||||
Self::ApplyVersion { .. }
|
||||
| Self::ApplyVersions { .. }
|
||||
| Self::Settle {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
@@ -2624,11 +2563,8 @@ impl DeleteRecoveryRpcOperation {
|
||||
pub enum DeleteRecoveryRpcReply {
|
||||
OrphanDataDir { plain: bool },
|
||||
Discovered(DeleteRecoveryPage),
|
||||
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
|
||||
Inspected(DeleteRecoveryInspection),
|
||||
InspectedBounded(BoundedDeleteRecoveryInspection),
|
||||
Settled(DeleteRecoveryInspection),
|
||||
SettledBounded(BoundedDeleteRecoverySettlementReply),
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
@@ -3106,8 +3042,6 @@ mod tests {
|
||||
recursive: true,
|
||||
incl_deleted: false,
|
||||
skip_hidden_prefix_check: false,
|
||||
hidden_entry_budget: 0,
|
||||
pending_prefix: None,
|
||||
report_notfound: false,
|
||||
filter_prefix: Some("prefix_".to_string()),
|
||||
forward_to: Some("object/path".to_string()),
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A candidate index never replaces multipart metadata or quorum validation.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
|
||||
pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
|
||||
|
||||
/// A bounded page from one disk's continuously maintained, boot-local index.
|
||||
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
|
||||
/// pages from different epochs or physical disk identities.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexPage {
|
||||
pub bucket: String,
|
||||
pub disk_id: Uuid,
|
||||
pub epoch: Uuid,
|
||||
pub roots: Vec<String>,
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
|
||||
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
|
||||
|
||||
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
|
||||
/// Old peers may ignore this field, but their ordinary response cannot decode
|
||||
/// as the required signed, challenge-bound MultipartIndexRpcReply.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcRequest {
|
||||
pub disk_id: String,
|
||||
pub metrics: bool,
|
||||
pub noop: bool,
|
||||
pub multipart_index: MultipartIndexQuery,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexQuery {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub bucket: String,
|
||||
pub marker: Option<String>,
|
||||
pub page_cap: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcReply {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub page: MultipartIndexPage,
|
||||
pub proof: Vec<u8>,
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcRequest {
|
||||
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
|
||||
Self {
|
||||
disk_id: String::new(),
|
||||
metrics: false,
|
||||
noop: true,
|
||||
multipart_index: MultipartIndexQuery {
|
||||
operation: "list_multipart_index_roots".into(),
|
||||
version: 1,
|
||||
challenge: Uuid::new_v4(),
|
||||
bucket: bucket.to_owned(),
|
||||
marker: marker.map(str::to_owned),
|
||||
page_cap: MULTIPART_INDEX_PAGE_SIZE,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> crate::disk::error::Result<()> {
|
||||
let query = &self.multipart_index;
|
||||
if !self.disk_id.is_empty()
|
||||
|| self.metrics
|
||||
|| !self.noop
|
||||
|| query.operation != "list_multipart_index_roots"
|
||||
|| query.version != 1
|
||||
|| query.challenge.is_nil()
|
||||
|| query.bucket.is_empty()
|
||||
|| query.bucket.len() > 255
|
||||
|| query.bucket.contains('/')
|
||||
|| matches!(query.bucket.as_str(), "." | "..")
|
||||
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|
||||
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::MethodNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
|
||||
}
|
||||
|
||||
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_root(root: &str) -> bool {
|
||||
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
impl MultipartIndexPage {
|
||||
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
if expected_disk.is_nil()
|
||||
|| self.disk_id != expected_disk
|
||||
|| self.epoch.is_nil()
|
||||
|| self.bucket != bucket
|
||||
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|
||||
|| self
|
||||
.roots
|
||||
.iter()
|
||||
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|
||||
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
|| self
|
||||
.next_marker
|
||||
.as_ref()
|
||||
.is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcReply {
|
||||
/// This is structural validation only; the caller must also verify `proof`.
|
||||
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
let query = &request.multipart_index;
|
||||
request.validate()?;
|
||||
if self.operation != query.operation
|
||||
|| self.version != query.version
|
||||
|| self.challenge != query.challenge
|
||||
|| self.proof.len() != 32
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
|
||||
}
|
||||
}
|
||||
|
||||
/// A disk can attest coverage only after every page belongs to the same physical
|
||||
/// identity and index generation. An interrupted/changed stream contributes no
|
||||
/// partial candidates to the erasure set's coverage count.
|
||||
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
|
||||
where
|
||||
F: FnMut(Option<String>) -> Fut,
|
||||
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
|
||||
{
|
||||
let mut epoch = None;
|
||||
let mut marker = None;
|
||||
let mut roots = Vec::new();
|
||||
loop {
|
||||
let page = fetch(marker.clone()).await?;
|
||||
page.validate(bucket, marker.as_deref(), identity)?;
|
||||
if epoch.is_some_and(|epoch| epoch != page.epoch) {
|
||||
return Err(crate::disk::error::DiskError::DiskNotFound);
|
||||
}
|
||||
epoch = Some(page.epoch);
|
||||
roots.extend(page.roots);
|
||||
marker = page.next_marker;
|
||||
if marker.is_none() {
|
||||
return Ok(roots);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
|
||||
let request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let id = Uuid::new_v4();
|
||||
let mut reply = MultipartIndexRpcReply {
|
||||
operation: request.multipart_index.operation.clone(),
|
||||
version: 1,
|
||||
challenge: request.multipart_index.challenge,
|
||||
page: MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec!["a".repeat(64)],
|
||||
next_marker: None,
|
||||
},
|
||||
proof: vec![0; 32],
|
||||
};
|
||||
reply.validate(&request, id).expect("valid structural envelope");
|
||||
assert!(
|
||||
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
|
||||
"old challenge cannot replay"
|
||||
);
|
||||
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
|
||||
reply.page.next_marker = Some("a".repeat(64));
|
||||
assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
|
||||
reply.page.next_marker = None;
|
||||
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
|
||||
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let mut request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let mut page = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: Uuid::new_v4(),
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let canonical = request.canonical_response("disk", &page).expect("canonical response");
|
||||
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
|
||||
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&request.canonical_response("disk", page).expect("canonical"),
|
||||
&proof,
|
||||
)
|
||||
};
|
||||
verify(&request, &page).expect("exact response");
|
||||
request.multipart_index.marker = Some("0".repeat(64));
|
||||
assert!(verify(&request, &page).is_err(), "cursor is signed");
|
||||
request.multipart_index.marker = None;
|
||||
page.epoch = Uuid::new_v4();
|
||||
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
for replace_disk in [false, true] {
|
||||
let first = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
|
||||
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
|
||||
};
|
||||
let second = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: if replace_disk { Uuid::new_v4() } else { id },
|
||||
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let mut replies = std::collections::VecDeque::from([first, second]);
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
let page = replies.pop_front().expect("bounded fixture pages");
|
||||
if marker.is_some() {
|
||||
assert!(page.roots.is_empty());
|
||||
}
|
||||
std::future::ready(Ok(page))
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
|
||||
assert!(replies.is_empty(), "exercise the continuation response");
|
||||
}
|
||||
}
|
||||
}
|
||||
+448
-769
File diff suppressed because it is too large
Load Diff
@@ -269,11 +269,6 @@ pub enum StorageError {
|
||||
/// retained for operator inspection.
|
||||
#[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
|
||||
DeleteRecoveryLegacyAmbiguous,
|
||||
/// Delete recovery exhausted the per-request work budget before it could
|
||||
/// classify every residue. Retrying later makes progress, so callers treat
|
||||
/// it like `SlowDown`; it stays distinct so the disk error round-trips.
|
||||
#[error("delete recovery work limit reached; please retry")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
#[error("Decommission not started")]
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
@@ -485,7 +480,6 @@ impl From<DiskError> for StorageError {
|
||||
DiskError::InvalidPath => StorageError::InvalidPath,
|
||||
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
|
||||
}
|
||||
}
|
||||
@@ -513,7 +507,6 @@ impl StorageError {
|
||||
StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
StorageError::StorageFull => DiskError::DiskFull,
|
||||
StorageError::SlowDown => DiskError::TooManyOpenFiles,
|
||||
StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
StorageError::InconsistentDisk => DiskError::InconsistentDisk,
|
||||
StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
|
||||
@@ -717,7 +710,6 @@ impl Clone for StorageError {
|
||||
StorageError::StorageFull => StorageError::StorageFull,
|
||||
StorageError::SlowDown => StorageError::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
|
||||
StorageError::InvalidUploadIDKeyCombination(a, b) => {
|
||||
StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
|
||||
@@ -860,7 +852,6 @@ impl StorageError {
|
||||
StorageError::StorageFull => StorageErrorCode::StorageFull,
|
||||
StorageError::SlowDown => StorageErrorCode::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
|
||||
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
|
||||
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
|
||||
@@ -967,7 +958,6 @@ impl StorageError {
|
||||
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
|
||||
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
|
||||
StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
|
||||
StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
|
||||
StorageErrorCode::PrefixAccessDenied => {
|
||||
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
|
||||
}
|
||||
|
||||
@@ -79,14 +79,6 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
|
||||
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
|
||||
}
|
||||
|
||||
/// Validate that replicated compression metadata can be read by this build.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an unsupported compression scheme.
|
||||
pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
|
||||
compression_scheme_to_read_plan(scheme).map(|_| ())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadEncryptionBackend {
|
||||
Legacy,
|
||||
@@ -452,12 +444,10 @@ impl WritePlan {
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
|
||||
WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
|
||||
// Each part write needs an independent fingerprint, including a
|
||||
// retry at the same part number. This salt affects only the ETag
|
||||
// MAC; encryption framing and its data nonce stay unchanged.
|
||||
WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
@@ -568,54 +558,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
|
||||
let plaintext = b"same confidential part";
|
||||
let size = i64::try_from(plaintext.len()).expect("test length");
|
||||
for object_key_mode in [false, true] {
|
||||
let mut tags = std::collections::HashSet::new();
|
||||
for part_number in [1, 2, 1] {
|
||||
let encryption = if object_key_mode {
|
||||
WriteEncryption::multipart_object_key([7; 32], part_number)
|
||||
} else {
|
||||
WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
|
||||
};
|
||||
let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(source, size)
|
||||
.expect("part plan");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
|
||||
let tag = reader.try_resolve_etag().expect("part fingerprint");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
|
||||
assert!(
|
||||
tags.insert(tag),
|
||||
"different part numbers and retries must not identify matching plaintext"
|
||||
);
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let mut decrypt = if object_key_mode {
|
||||
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
|
||||
} else {
|
||||
DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
|
||||
};
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let mut decrypt = DecryptReader::new_multipart(
|
||||
Cursor::new(ciphertext),
|
||||
[7; 32],
|
||||
if object_key_mode { [0; 12] } else { [9; 12] },
|
||||
vec![part_number as usize],
|
||||
);
|
||||
let mut decoded = Vec::new();
|
||||
decrypt
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("existing data nonce still decrypts");
|
||||
assert_eq!(decoded, plaintext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -28,6 +28,7 @@ use tokio_util::sync::CancellationToken;
|
||||
#[derive(Clone)]
|
||||
pub struct NamespaceLockFence {
|
||||
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
|
||||
local_probes: Arc<Vec<rustfs_lock::fast_lock::guard::FastLockGuardProbe>>,
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
|
||||
}
|
||||
@@ -44,6 +45,7 @@ impl NamespaceLockFence {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
local_probes: Arc::default(),
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
|
||||
}
|
||||
@@ -58,17 +60,28 @@ impl NamespaceLockFence {
|
||||
{
|
||||
return true;
|
||||
}
|
||||
self.signals.iter().any(|signal| signal.is_lost())
|
||||
self.signals.iter().any(|signal| signal.is_lost()) || self.local_probes.iter().any(|probe| probe.is_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn add_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
|
||||
Arc::make_mut(&mut self.signals).push(signal);
|
||||
}
|
||||
|
||||
fn add_local_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
if let rustfs_lock::NamespaceLockGuard::Fast(guard) = guard
|
||||
&& let Some(probe) = guard.release_probe()
|
||||
{
|
||||
Arc::make_mut(&mut self.local_probes).push(probe);
|
||||
}
|
||||
}
|
||||
|
||||
fn extend(&mut self, other: &Self) {
|
||||
if !Arc::ptr_eq(&self.signals, &other.signals) {
|
||||
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
|
||||
}
|
||||
if !Arc::ptr_eq(&self.local_probes, &other.local_probes) {
|
||||
Arc::make_mut(&mut self.local_probes).extend(other.local_probes.iter().cloned());
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
|
||||
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
|
||||
@@ -106,6 +119,7 @@ impl NamespaceLockSignalTestFence {
|
||||
) -> Self {
|
||||
let fence = NamespaceLockFence {
|
||||
signals: Arc::default(),
|
||||
local_probes: Arc::default(),
|
||||
forced_lost: Arc::new(vec![loss_handle]),
|
||||
};
|
||||
let signal_key = Arc::as_ptr(signal) as usize;
|
||||
@@ -160,6 +174,20 @@ pub struct ObjectLockConfigSnapshot {
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
|
||||
self._lifecycle_guard.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
|
||||
self.bucket.as_deref() == Some(bucket)
|
||||
&& incarnation.is_some()
|
||||
&& self.bucket_incarnation_id == incarnation
|
||||
&& self
|
||||
._lifecycle_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
|
||||
Self {
|
||||
store_id: None,
|
||||
@@ -208,6 +236,7 @@ impl ObjectLockConfigSnapshot {
|
||||
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
|
||||
lifecycle_fence.add_signal(signal);
|
||||
}
|
||||
lifecycle_fence.add_local_guard(&lifecycle_guard);
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
bucket: Some(bucket.to_string()),
|
||||
@@ -949,81 +978,8 @@ pub enum ShardIntegrityWriteMode {
|
||||
Protected,
|
||||
}
|
||||
|
||||
/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
|
||||
/// Credits cover directory entries (including a minimum charge for an empty probe) and
|
||||
/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
|
||||
#[derive(Debug)]
|
||||
pub struct DeleteRecoveryWorkBudget {
|
||||
remaining: std::sync::atomic::AtomicU32,
|
||||
}
|
||||
|
||||
/// Entries one disk may visit per recovery step (discovery plus inspection).
|
||||
/// Every disk of every set a request fans out to is admitted up to this much
|
||||
/// and charged on completion, so a budget must cover that many disks.
|
||||
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
|
||||
|
||||
impl Default for DeleteRecoveryWorkBudget {
|
||||
/// Covers a full 16-drive set with headroom; `for_disks` scales it when a
|
||||
/// request spans more drives than that.
|
||||
fn default() -> Self {
|
||||
Self::new(16 * 1024 * 1024)
|
||||
}
|
||||
}
|
||||
|
||||
impl DeleteRecoveryWorkBudget {
|
||||
/// A per-request budget sized for `disks` concurrent up-front reservations.
|
||||
pub(crate) fn for_disks(disks: usize) -> Self {
|
||||
let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
|
||||
let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
|
||||
let default = u64::from(Self::default().remaining());
|
||||
Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
|
||||
}
|
||||
|
||||
pub(crate) fn new(entries: u32) -> Self {
|
||||
Self {
|
||||
remaining: std::sync::atomic::AtomicU32::new(entries),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
|
||||
use std::sync::atomic::Ordering;
|
||||
self.remaining
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
|
||||
(remaining > 0).then(|| remaining.saturating_sub(maximum))
|
||||
})
|
||||
.map(|remaining| remaining.min(maximum))
|
||||
.map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
|
||||
pub(crate) fn remaining(&self) -> u32 {
|
||||
self.remaining.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Admit one bounded disk call: refuse once the budget is spent, otherwise
|
||||
/// cap the call at `maximum` or what remains. The work actually performed is
|
||||
/// charged on completion with `charge`, so concurrent calls never exhaust
|
||||
/// the budget by reservation alone (a bulk delete fans out dozens of
|
||||
/// inspections per key); each call is still bounded by its own cap.
|
||||
pub(crate) fn admit(&self, maximum: u32) -> crate::disk::error::Result<u32> {
|
||||
match self.remaining() {
|
||||
0 => Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit),
|
||||
remaining => Ok(remaining.min(maximum)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the entries a bounded call visited (at least one per call).
|
||||
pub(crate) fn charge(&self, visited: u32) {
|
||||
use std::sync::atomic::Ordering;
|
||||
let _ = self.remaining.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
|
||||
Some(remaining.saturating_sub(visited.max(1)))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
#[doc(hidden)]
|
||||
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
pub max_parity: bool,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -1175,12 +1131,6 @@ pub struct ObjectOptions {
|
||||
pub expiration: ExpirationOptions,
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
/// In-process request gate evaluated on authoritative completion metadata.
|
||||
/// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
|
||||
/// implementations may own customer encryption keys.
|
||||
#[doc(hidden)]
|
||||
pub multipart_completion_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
/// Internal compare-and-set condition for replication workers publishing
|
||||
/// terminal status after remote I/O. Storage validates it while holding
|
||||
@@ -1194,10 +1144,6 @@ pub struct ObjectOptions {
|
||||
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
|
||||
/// before a destructive commit reaches the set layer.
|
||||
pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
|
||||
/// ECStore single PUTs acquire their destructive-write configuration guards
|
||||
/// only after staging the client body. External lock owners supply a snapshot.
|
||||
#[doc(hidden)]
|
||||
pub defer_put_object_lock_snapshot: bool,
|
||||
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
@@ -1229,11 +1175,6 @@ pub enum ReplicationStatusWritebackMode {
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
|
||||
self.delete_recovery_work_budget
|
||||
.get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
|
||||
}
|
||||
|
||||
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
|
||||
match self.shard_integrity_write_mode {
|
||||
Some(ShardIntegrityWriteMode::Protected) => true,
|
||||
@@ -1441,6 +1382,13 @@ impl ObjectOptions {
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
self.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
if let rustfs_lock::NamespaceLockGuard::Fast(local) = guard
|
||||
&& !local.is_disabled()
|
||||
{
|
||||
self.namespace_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
.add_local_guard(guard);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
@@ -1448,6 +1396,7 @@ impl ObjectOptions {
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
fence.add_signal(signal);
|
||||
}
|
||||
fence.add_local_guard(guard);
|
||||
}
|
||||
|
||||
pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
|
||||
|
||||
@@ -58,7 +58,7 @@ use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
@@ -71,6 +71,9 @@ const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
|
||||
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
|
||||
|
||||
// Bound abandoned speculative RPCs independently for each storage instance.
|
||||
pub(crate) const SPECULATIVE_RESERVATION_LIMIT: usize = 64;
|
||||
|
||||
/// A lease is deliberately short-lived. The coordinator treats expiry as a
|
||||
/// failed publication rather than silently continuing with a peer that may
|
||||
/// have started movement after the lease was abandoned.
|
||||
@@ -101,6 +104,12 @@ pub(crate) struct NamespaceCommitGuard {
|
||||
counted: bool,
|
||||
}
|
||||
|
||||
impl NamespaceCommitGuard {
|
||||
pub(crate) fn covers_instance(&self, ctx: &Arc<InstanceContext>) -> bool {
|
||||
Arc::ptr_eq(&self.ctx, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NamespaceCommitGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.counted {
|
||||
@@ -332,6 +341,7 @@ pub struct InstanceContext {
|
||||
scanner_publication_state: AtomicU8,
|
||||
namespace_commits: AtomicU64,
|
||||
namespace_commit_generation: AtomicU64,
|
||||
speculative_reservations: Arc<Semaphore>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
delete_recovery_hints: DeleteRecoveryHints,
|
||||
@@ -392,6 +402,7 @@ impl InstanceContext {
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
namespace_commits: AtomicU64::new(0),
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
speculative_reservations: Arc::new(Semaphore::new(SPECULATIVE_RESERVATION_LIMIT)),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
delete_recovery_hints: DeleteRecoveryHints::default(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -413,6 +424,11 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
/// Skip speculative work when its detached owners already fill the instance budget.
|
||||
pub(crate) fn try_acquire_speculative_reservation(&self) -> Option<OwnedSemaphorePermit> {
|
||||
Arc::clone(&self.speculative_reservations).try_acquire_owned().ok()
|
||||
}
|
||||
|
||||
/// Run a multi-step mutation on its own task and wait for its output.
|
||||
///
|
||||
/// Dropping the returned future leaves the task running to completion, so
|
||||
@@ -1143,6 +1159,26 @@ mod tests {
|
||||
assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speculative_reservation_budget_is_bounded_and_instance_local() {
|
||||
let first = InstanceContext::new();
|
||||
let other = InstanceContext::new();
|
||||
let mut permits = (0..super::SPECULATIVE_RESERVATION_LIMIT)
|
||||
.map(|_| first.try_acquire_speculative_reservation().expect("initial permit"))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(first.try_acquire_speculative_reservation().is_none());
|
||||
assert!(
|
||||
other.try_acquire_speculative_reservation().is_some(),
|
||||
"budgets belong to their own instance"
|
||||
);
|
||||
drop(permits.pop());
|
||||
let replacement = first
|
||||
.try_acquire_speculative_reservation()
|
||||
.expect("dropping an owner frees its slot");
|
||||
assert!(first.try_acquire_speculative_reservation().is_none());
|
||||
drop(replacement);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
|
||||
let first = Arc::new(InstanceContext::new());
|
||||
@@ -1151,6 +1187,8 @@ mod tests {
|
||||
other.set_scanner_publication_state(false);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
let one = first.begin_namespace_commit();
|
||||
assert!(one.covers_instance(&first));
|
||||
assert!(!one.covers_instance(&other));
|
||||
let shared_owner = Arc::clone(&one);
|
||||
let two = first.begin_namespace_commit();
|
||||
assert!(first.namespace_commits_pending());
|
||||
|
||||
@@ -318,21 +318,6 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
|
||||
match rebalance_error_source(err) {
|
||||
Error::Lock(err) => is_rebalance_transient_lock_error(err),
|
||||
// No metadata was written when lock admission failed. Each attempt
|
||||
// must acquire its quorum again before loading or saving this run.
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket,
|
||||
object,
|
||||
..
|
||||
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return true;
|
||||
@@ -400,7 +385,9 @@ where
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(err) => {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1)
|
||||
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -732,82 +719,6 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
|
||||
let mut attempts = 0;
|
||||
retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(if attempts == 1 {
|
||||
resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)),
|
||||
"stop_rebalance",
|
||||
)
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("a later quorum-protected admission must be allowed to succeed");
|
||||
assert_eq!(attempts, 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
|
||||
for max_attempts in [0, 1, 3] {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, max_attempts, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)))
|
||||
})
|
||||
.await;
|
||||
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
|
||||
assert_eq!(attempts, max_attempts.max(1));
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&error),
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(if attempts == 1 {
|
||||
rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)
|
||||
} else {
|
||||
Error::OperationCanceled
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
|
||||
assert!(matches!(result, Err(Error::OperationCanceled)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
|
||||
for err in [
|
||||
@@ -820,20 +731,6 @@ mod error_source_tests {
|
||||
required: 3,
|
||||
achieved: 2,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: "data-bucket".to_string(),
|
||||
object: REBAL_META_NAME.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
|
||||
object: "other-metadata.bin".to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::other("stale rebalance run rejected: lock acquisition timed out"),
|
||||
Error::other("rebalance distributed run fence lost"),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
|
||||
@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
bytes.len()
|
||||
};
|
||||
|
||||
Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
|
||||
@@ -37,6 +37,7 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::{
|
||||
select,
|
||||
sync::{Notify, RwLock},
|
||||
@@ -8182,7 +8183,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
@@ -8247,7 +8248,7 @@ mod tests {
|
||||
_opts: WarmBackendGetOpts,
|
||||
) -> std::result::Result<ReadCloser, std::io::Error> {
|
||||
if self.healthy {
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
} else {
|
||||
Err(std::io::Error::other("mock get failed"))
|
||||
}
|
||||
@@ -9894,7 +9895,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
|
||||
@@ -2538,6 +2538,8 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
|
||||
pub(in crate::set_disk) tail_has_owned_bucket_fences: bool,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
@@ -3468,6 +3470,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
|
||||
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
@@ -3483,6 +3487,8 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: None,
|
||||
multipart_proof_authority: None,
|
||||
multipart_commit_owner: None,
|
||||
reservation_ctx: None,
|
||||
prepared_reservations: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3519,6 +3525,415 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_reservation_context(
|
||||
mut self,
|
||||
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
) -> Self {
|
||||
self.reservation_ctx = ctx;
|
||||
self.prepared_reservations = reservations;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MetadataRenewalTarget {
|
||||
disk: DiskStore,
|
||||
token: Uuid,
|
||||
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
|
||||
}
|
||||
|
||||
struct MetadataRenewalGroup {
|
||||
targets: Arc<parking_lot::Mutex<HashMap<usize, MetadataRenewalTarget>>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Drop for MetadataRenewalGroup {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameMetadataReservation {
|
||||
disk: DiskStore,
|
||||
token: Option<Uuid>,
|
||||
// Remote reservation protocol v2 and the local owned path both retain
|
||||
// bucket publication and lifecycle guards until the physical write drains.
|
||||
owns_bucket_fence: bool,
|
||||
short: bool,
|
||||
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
|
||||
renewal: Option<(Arc<MetadataRenewalGroup>, usize)>,
|
||||
}
|
||||
|
||||
impl RenameMetadataReservation {
|
||||
fn stop_renewal(&mut self) {
|
||||
if let Some((group, index)) = self.renewal.take() {
|
||||
let mut targets = group.targets.lock();
|
||||
targets.remove(&index);
|
||||
if targets.is_empty() {
|
||||
group.task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cancel(&mut self) -> disk::error::Result<()> {
|
||||
self.stop_renewal();
|
||||
let Some(token) = self.token else { return Ok(()) };
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
|
||||
Ok(Ok(true)) => {
|
||||
self.token = None;
|
||||
Ok(())
|
||||
}
|
||||
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameMetadataReservation {
|
||||
fn drop(&mut self) {
|
||||
self.stop_renewal();
|
||||
let Some(token) = self.token.take() else { return };
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let disk = self.disk.clone();
|
||||
runtime.spawn(async move {
|
||||
let _ = disk.cancel_object_metadata_reservation(token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum RenameReservationMode {
|
||||
Ownerless,
|
||||
Owned,
|
||||
ShortOwned,
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
owns_bucket_fence: bool,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
prepare_rename_metadata_reservations_until_abandoned(
|
||||
disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
ctx,
|
||||
if owns_bucket_fence {
|
||||
RenameReservationMode::Owned
|
||||
} else {
|
||||
RenameReservationMode::Ownerless
|
||||
},
|
||||
std::future::pending(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations_until_abandoned(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
mode: RenameReservationMode,
|
||||
abandoned: impl std::future::Future<Output = ()>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
let owns_bucket_fence = !matches!(mode, RenameReservationMode::Ownerless);
|
||||
let short = matches!(mode, RenameReservationMode::ShortOwned);
|
||||
let mut abandoned = Box::pin(abandoned);
|
||||
let discarded = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let mut pending: FuturesUnordered<_> = disks
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(_index, disk)| {
|
||||
let discarded = Arc::clone(&discarded);
|
||||
async move {
|
||||
let result: disk::error::Result<Option<RenameMetadataReservation>> = async {
|
||||
if discarded.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(disk) = disk else { return Ok(None) };
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_BEFORE).await;
|
||||
if disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, disk, None, false).await?;
|
||||
}
|
||||
if discarded.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Ok(None);
|
||||
}
|
||||
let prepared_at = tokio::time::Instant::now();
|
||||
let token = if disk.is_local() && owns_bucket_fence {
|
||||
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or(DiskError::DiskOngoingReq)?;
|
||||
if short {
|
||||
store
|
||||
.reserve_local_object_metadata_short(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_millis(50),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
} else if short {
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
remote
|
||||
.reserve_object_metadata_short(bucket, object, incarnation, Duration::from_millis(50))
|
||||
.await?
|
||||
} else {
|
||||
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
|
||||
.await?
|
||||
};
|
||||
// No await separates receipt from installing the cancellation owner.
|
||||
let reservation = RenameMetadataReservation {
|
||||
disk: disk.clone(),
|
||||
token: Some(token),
|
||||
owns_bucket_fence: !disk.is_local() || owns_bucket_fence,
|
||||
short,
|
||||
renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)),
|
||||
renewal: None,
|
||||
};
|
||||
if short {
|
||||
// Install the cancellation owner before extending the short token.
|
||||
// A dropped renew response still leaves us the token to cancel.
|
||||
disk.renew_object_metadata(bucket, object, token, Duration::from_millis(50))
|
||||
.await?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_AFTER).await;
|
||||
Ok(Some(reservation))
|
||||
}
|
||||
.await;
|
||||
(_index, result)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut reservations: Vec<Option<RenameMetadataReservation>> = (0..disks.len()).map(|_| None).collect();
|
||||
let mut discard = false;
|
||||
while !pending.is_empty() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut abandoned, if !discard => {
|
||||
discard = true;
|
||||
discarded.store(true, std::sync::atomic::Ordering::Release);
|
||||
reservations.clear();
|
||||
}
|
||||
result = pending.next() => {
|
||||
let Some((index, result)) = result else { break };
|
||||
if discard {
|
||||
drop(result);
|
||||
} else {
|
||||
match result {
|
||||
Ok(reservation) => reservations[index] = reservation,
|
||||
Err(_) => {
|
||||
discard = true;
|
||||
discarded.store(true, std::sync::atomic::Ordering::Release);
|
||||
reservations.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if discard {
|
||||
// Drain all in-flight preparations so late tokens get a cancellation owner.
|
||||
return Ok(None);
|
||||
}
|
||||
let targets = Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
for (index, reservation) in reservations.iter().enumerate() {
|
||||
if let Some(reservation) = reservation {
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
targets.lock().insert(
|
||||
index,
|
||||
MetadataRenewalTarget {
|
||||
disk: reservation.disk.clone(),
|
||||
token,
|
||||
renewed_at: Arc::clone(&reservation.renewed_at),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
if !targets.lock().is_empty() {
|
||||
let renewal_targets = Arc::clone(&targets);
|
||||
let renewal_ctx = Arc::clone(ctx);
|
||||
let renewal_bucket = bucket.to_owned();
|
||||
let renewal_object = object.to_owned();
|
||||
// Short leases remain short even after renewal, so a lost renewal and
|
||||
// cancellation cannot strand a bucket fence for the legacy lease time.
|
||||
let task = tokio::spawn(async move {
|
||||
let interval = if short {
|
||||
Duration::from_millis(500)
|
||||
} else {
|
||||
Duration::from_secs(20)
|
||||
};
|
||||
let retry = if short {
|
||||
Duration::from_millis(250)
|
||||
} else {
|
||||
Duration::from_secs(5)
|
||||
};
|
||||
let lifetime = if short {
|
||||
Duration::from_secs(2)
|
||||
} else {
|
||||
Duration::from_secs(60)
|
||||
};
|
||||
let mut delay = interval;
|
||||
loop {
|
||||
tokio::time::sleep(delay).await;
|
||||
let current: Vec<_> = renewal_targets.lock().values().cloned().collect();
|
||||
if current.is_empty() {
|
||||
break;
|
||||
}
|
||||
// Once any acknowledgment is too old, the complete fanout can
|
||||
// no longer be safely activated. Stop retrying stale peers;
|
||||
// their receiver-side tokens still expire on their own.
|
||||
if current.iter().any(|target| target.renewed_at.lock().elapsed() >= lifetime) {
|
||||
break;
|
||||
}
|
||||
let results = join_all(current.into_iter().map(|target| {
|
||||
let ctx = Arc::clone(&renewal_ctx);
|
||||
let bucket = &renewal_bucket;
|
||||
let object = &renewal_object;
|
||||
async move {
|
||||
let started = tokio::time::Instant::now();
|
||||
let admitted = !target.disk.is_local()
|
||||
|| crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok();
|
||||
if admitted
|
||||
&& target
|
||||
.disk
|
||||
.renew_object_metadata(bucket, object, target.token, retry)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
*target.renewed_at.lock() = started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
delay = if results.into_iter().all(|success| success) {
|
||||
interval
|
||||
} else {
|
||||
retry
|
||||
};
|
||||
}
|
||||
});
|
||||
let group = Arc::new(MetadataRenewalGroup { targets, task });
|
||||
for (index, reservation) in reservations.iter_mut().enumerate() {
|
||||
if let Some(reservation) = reservation {
|
||||
reservation.renewal = Some((Arc::clone(&group), index));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
pub(crate) fn align_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<Vec<Option<RenameMetadataReservation>>> {
|
||||
if reservations.len() != disks.len() {
|
||||
return None;
|
||||
}
|
||||
let mut aligned = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
aligned.push(None);
|
||||
continue;
|
||||
};
|
||||
let index = reservations
|
||||
.iter()
|
||||
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
|
||||
aligned.push(reservations[index].take());
|
||||
}
|
||||
// Disks lost while staging need no activation; dropping their still-inert
|
||||
// reservations schedules cancellation without blocking the commit.
|
||||
Some(aligned)
|
||||
}
|
||||
|
||||
async fn activate_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
// Short tokens continue renewing during activation; an activated token
|
||||
// can otherwise expire while another disk's activation is still pending.
|
||||
// Long tokens retain their existing non-renewable queued behavior.
|
||||
for reservation in reservations.iter_mut().flatten() {
|
||||
if !reservation.short {
|
||||
reservation.stop_renewal();
|
||||
}
|
||||
}
|
||||
if reservations.iter().flatten().any(|reservation| {
|
||||
let max_age = if reservation.short {
|
||||
Duration::from_secs(1)
|
||||
} else {
|
||||
Duration::from_secs(50)
|
||||
};
|
||||
reservation.renewed_at.lock().elapsed() >= max_age
|
||||
}) {
|
||||
return Ok(None);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_reservations = &reservations;
|
||||
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
|
||||
if reservation.disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
|
||||
}
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
let result = reservation
|
||||
.disk
|
||||
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
if result.is_ok() {
|
||||
let index = test_reservations
|
||||
.iter()
|
||||
.position(|candidate| {
|
||||
candidate
|
||||
.as_ref()
|
||||
.is_some_and(|candidate| std::ptr::eq(candidate, reservation))
|
||||
})
|
||||
.expect("activated reservation belongs to the fanout");
|
||||
rename_fanout_barrier::checkpoint(object, index, rename_fanout_barrier::PHASE_RESERVATION_ACTIVATED).await;
|
||||
}
|
||||
result
|
||||
}))
|
||||
.await;
|
||||
for reservation in reservations.iter_mut().flatten() {
|
||||
reservation.stop_renewal();
|
||||
}
|
||||
if activated.into_iter().any(|result| result.is_err()) {
|
||||
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
|
||||
if cancelled.into_iter().any(|result| result.is_err()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -3530,7 +3945,7 @@ type RenameDataLegacyTuple = (
|
||||
Option<OldCurrentSize>,
|
||||
);
|
||||
|
||||
fn put_rename_early_ack_enabled() -> bool {
|
||||
pub(crate) fn put_rename_early_ack_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
|
||||
}
|
||||
|
||||
@@ -3896,6 +4311,8 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
tail_has_owned_bucket_fences: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3992,6 +4409,8 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority: _,
|
||||
multipart_commit_owner: _,
|
||||
reservation_ctx,
|
||||
prepared_reservations,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4007,6 +4426,21 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
|
||||
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
|
||||
(_, _) => None,
|
||||
};
|
||||
let tail_has_metadata_reservations = reservations.is_some();
|
||||
let tail_has_owned_bucket_fences = reservations.as_ref().is_some_and(|reservations| {
|
||||
reservations.len() == disks.len()
|
||||
&& disks.iter().any(Option::is_some)
|
||||
&& disks.iter().zip(reservations).all(|(disk, reservation)| match disk {
|
||||
Some(disk) => reservation
|
||||
.as_ref()
|
||||
.is_some_and(|owner| owner.owns_bucket_fence && Arc::ptr_eq(&owner.disk, disk)),
|
||||
None => true,
|
||||
})
|
||||
});
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -4039,6 +4473,11 @@ impl SetDisks {
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let reservation = reservations
|
||||
.as_mut()
|
||||
.and_then(|entries| entries.get_mut(i))
|
||||
.and_then(Option::take);
|
||||
let reservation_ctx = reservation_ctx.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4049,6 +4488,17 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_admission = if reservation.is_some() && disk.is_local() {
|
||||
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
|
||||
// The fanout owner already covers this instance through
|
||||
// physical completion; only add a disk owner when it does not.
|
||||
let covered = namespace_commit_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.covers_instance(ctx));
|
||||
crate::store::admit_local_disk(ctx, &disk, None, !covered).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
@@ -4074,6 +4524,11 @@ impl SetDisks {
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
|
||||
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
|
||||
} else {
|
||||
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
|
||||
};
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4083,9 +4538,8 @@ impl SetDisks {
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
|
||||
namespace_owner,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4363,6 +4817,8 @@ impl SetDisks {
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
|
||||
commit.tail_has_owned_bucket_fences = tail_has_owned_bucket_fences;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
@@ -4432,6 +4888,8 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority,
|
||||
multipart_commit_owner,
|
||||
reservation_ctx: _,
|
||||
prepared_reservations: _,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4901,6 +5359,8 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
tail_has_owned_bucket_fences: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7358,6 +7818,9 @@ pub(crate) mod rename_fanout_barrier {
|
||||
MetadataSlowtailFault,
|
||||
NonInlineHedgeTimer,
|
||||
LateMetadataRefresh,
|
||||
ReservationBefore,
|
||||
ReservationAfter,
|
||||
ReservationActivated,
|
||||
}
|
||||
|
||||
impl From<&'static str> for Phase {
|
||||
@@ -7380,6 +7843,9 @@ pub(crate) mod rename_fanout_barrier {
|
||||
pub const PHASE_ROLLBACK: Phase = Phase::Rollback;
|
||||
pub const PHASE_READ_VERSION: Phase = Phase::ReadVersion;
|
||||
pub const PHASE_METADATA_SLOWTAIL_FAULT: Phase = Phase::MetadataSlowtailFault;
|
||||
pub const PHASE_RESERVATION_BEFORE: Phase = Phase::ReservationBefore;
|
||||
pub const PHASE_RESERVATION_AFTER: Phase = Phase::ReservationAfter;
|
||||
pub const PHASE_RESERVATION_ACTIVATED: Phase = Phase::ReservationActivated;
|
||||
|
||||
/// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk.
|
||||
pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer;
|
||||
@@ -7454,6 +7920,18 @@ pub(crate) mod rename_fanout_barrier {
|
||||
}
|
||||
|
||||
impl BarrierHandle {
|
||||
/// Keep the current task parked without intercepting later tasks.
|
||||
pub fn disarm(&self) {
|
||||
let mut reg = lock();
|
||||
if reg
|
||||
.armed
|
||||
.get(&self.key)
|
||||
.is_some_and(|armed| Arc::ptr_eq(&armed.release, &self.release))
|
||||
{
|
||||
reg.armed.remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Await until the armed fan-out task has parked at the checkpoint. Uses a
|
||||
/// stored-permit `Notify`, so this is race-free regardless of whether the
|
||||
/// task reaches the checkpoint before or after this call — no sleeps.
|
||||
@@ -7619,6 +8097,45 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() {
|
||||
let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await;
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(endpoint.clone(), Some(disk.clone()));
|
||||
let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false)
|
||||
.await
|
||||
.expect("prepare")
|
||||
.expect("complete fanout");
|
||||
let group = Arc::clone(
|
||||
&reservations[0]
|
||||
.as_ref()
|
||||
.expect("reservation")
|
||||
.renewal
|
||||
.as_ref()
|
||||
.expect("renewal")
|
||||
.0,
|
||||
);
|
||||
ctx.local_disk_map().write().await.remove(&endpoint);
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(21)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(41)).await;
|
||||
for _ in 0..10 {
|
||||
if group.task.is_finished() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::resume();
|
||||
assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts");
|
||||
drop(reservations);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() {
|
||||
use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint};
|
||||
|
||||
@@ -195,14 +195,7 @@ impl SetDisks {
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
// With no decodable version, the per-version quorum loop cannot
|
||||
// reject corrupt or unavailable replicas. Only complete, readable
|
||||
// evidence may establish that this key has no authoritative version.
|
||||
return if complete {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
};
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
|
||||
@@ -357,7 +357,8 @@ use crate::io_support::rio::HashReader;
|
||||
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||
pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
|
||||
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
|
||||
|
||||
@@ -409,6 +410,10 @@ impl ObjectLockDiagGuard {
|
||||
self
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
self.guard.downgrade_committed_tail().await
|
||||
}
|
||||
|
||||
/// Whether the underlying namespace lock's heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
|
||||
/// point on this so a stale lock holder does not race a double-write.
|
||||
@@ -901,6 +906,10 @@ mod core;
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::{
|
||||
RenameMetadataReservation, RenameReservationMode, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
|
||||
prepare_rename_metadata_reservations_until_abandoned, put_rename_early_ack_enabled,
|
||||
};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
@@ -927,6 +936,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::pause_put_object_commit;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
@@ -4695,17 +4706,6 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
// Reusing a held writer additionally requires the same quorum, so repeated
|
||||
// clients must not give different weight to an otherwise identical domain.
|
||||
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
!left.is_empty()
|
||||
&& left.len() == right.len()
|
||||
&& left.iter().all(|client| {
|
||||
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
})
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -5655,9 +5655,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
// Distributed lock clients receive the raw ObjectKey; the pool
|
||||
// namespace is not part of the key stored by their backends.
|
||||
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -9892,7 +9892,7 @@ mod tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
@@ -9931,25 +9931,11 @@ mod tests {
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pools using identical backends lock the same raw object key"
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
|
||||
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
|
||||
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
|
||||
assert!(!same_distributed_lock_quorum(
|
||||
&[a.clone(), a.clone(), b.clone()],
|
||||
&[a.clone(), b.clone(), b]
|
||||
));
|
||||
assert!(!same_distributed_lock_quorum(&[], &[]));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
|
||||
@@ -141,7 +141,6 @@ pub(crate) struct DeleteRecoverySummary {
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
pub incarnation: Option<Uuid>,
|
||||
pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
pub dry_run: bool,
|
||||
pub explicit_delete: bool,
|
||||
/// Only the locked heal path may repair a live marker-creation source.
|
||||
@@ -157,7 +156,6 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
|
||||
Self {
|
||||
incarnation,
|
||||
work_budget: None,
|
||||
dry_run: false,
|
||||
explicit_delete: false,
|
||||
repair_marker_source: false,
|
||||
@@ -232,54 +230,19 @@ async fn inspect(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Vec<Result<DeleteRecoveryInspection>> {
|
||||
join_all(disks.iter().map(|disk| async move {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let state = if let Some(budget) = budget {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let reply = disk
|
||||
.inspect_delete_transaction_bounded(bucket, object, transaction, admitted)
|
||||
.await?;
|
||||
if reply.visited > admitted {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(reply.visited);
|
||||
reply.inspection
|
||||
} else {
|
||||
disk.inspect_delete_transaction(bucket, object, transaction).await?
|
||||
};
|
||||
let state = disk
|
||||
.as_ref()
|
||||
.ok_or(DiskError::DiskNotFound)?
|
||||
.inspect_delete_transaction(bucket, object, transaction)
|
||||
.await?;
|
||||
validate(&state, object, transaction)?;
|
||||
Ok(state)
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_bounded(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
budget: &crate::object_api::DeleteRecoveryWorkBudget,
|
||||
) -> Result<Vec<DeleteTransactionId>> {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
|
||||
let page = disk.discover_delete_transactions_bounded(bucket, object, admitted).await?;
|
||||
if page.visited > admitted
|
||||
|| page.transactions.len() > 256
|
||||
|| page
|
||||
.transactions
|
||||
.iter()
|
||||
.any(|id| id.object != object || id.transaction.is_nil())
|
||||
|| (!page.complete && !page.transactions.is_empty())
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(page.visited);
|
||||
if !page.complete {
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
Ok(page.transactions)
|
||||
}
|
||||
|
||||
async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
|
||||
let mut cursor = None;
|
||||
let mut ids = BTreeSet::new();
|
||||
@@ -310,92 +273,30 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RecoveryWorkScope<'a> {
|
||||
namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
|
||||
budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
}
|
||||
|
||||
async fn settle_outcome(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
|
||||
use crate::disk::DeleteRecoverySettlementOutcome;
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let outcome = if let Some(budget) = scope.budget {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
|
||||
let reply = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: state.id.object.clone(),
|
||||
transaction: state.id.transaction,
|
||||
expected_fingerprint: state.fingerprint,
|
||||
action,
|
||||
max_entries: admitted,
|
||||
},
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?;
|
||||
if reply.visited > admitted
|
||||
|| (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(reply.visited);
|
||||
reply.outcome
|
||||
} else {
|
||||
DeleteRecoverySettlementOutcome::Inspected(Box::new(
|
||||
disk.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?,
|
||||
))
|
||||
};
|
||||
if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
|
||||
validate(next, &state.id.object, state.id.transaction)?;
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn settle(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
|
||||
crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
|
||||
crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
|
||||
outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
|
||||
}
|
||||
|
||||
fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
|
||||
match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
|
||||
Err(_) => false,
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
namespace_owner.cloned(),
|
||||
)
|
||||
.await;
|
||||
let next = result?;
|
||||
validate(&next, &state.id.object, state.id.transaction)?;
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// Each phase fans out across independent drives, and awaits all started work
|
||||
@@ -406,16 +307,23 @@ async fn settle_group(
|
||||
copies: &mut [DeleteRecoveryInspection],
|
||||
action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
removed_versions: &mut HashSet<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
|
||||
let action = action(copy);
|
||||
async move {
|
||||
match action {
|
||||
Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
|
||||
.await
|
||||
.map(Some),
|
||||
Some(action) => settle(
|
||||
disk.as_ref().ok_or(DiskError::DiskNotFound)?,
|
||||
bucket,
|
||||
copy,
|
||||
action,
|
||||
can_commit,
|
||||
namespace_owner,
|
||||
)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
@@ -531,31 +439,16 @@ impl SetDisks {
|
||||
true
|
||||
}
|
||||
};
|
||||
let mut fast_receipt_quorum = false;
|
||||
if can_try_local_fast_path && can_commit() {
|
||||
let fast_published = join_all(disks.iter().map(|disk| async {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
disk.try_publish_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
context.namespace_owner.clone(),
|
||||
context.work_budget.clone(),
|
||||
)
|
||||
.await
|
||||
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
|
||||
if fast_receipt_quorum
|
||||
&& fast_published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let all_fast_published = fast_published.len() == self.set_drive_count
|
||||
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
|
||||
if all_fast_published && can_commit() {
|
||||
@@ -563,7 +456,6 @@ impl SetDisks {
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
|
||||
let ns_owner = context.namespace_owner.clone();
|
||||
let work_budget = context.work_budget.clone();
|
||||
async move {
|
||||
if !can_commit() {
|
||||
return Ok(false);
|
||||
@@ -574,24 +466,11 @@ impl SetDisks {
|
||||
let Ok(Some((journal, _, _))) = pub_res else {
|
||||
return Ok(false);
|
||||
};
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
ns_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
if finalized
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if finalized.iter().all(|res| matches!(res, Ok(true))) {
|
||||
if let Some(ticket) = context.recovery_ticket.as_ref() {
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
@@ -600,17 +479,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
|
||||
if states
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return if fast_receipt_quorum {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
};
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction).await;
|
||||
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
|
||||
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
|
||||
&& states.iter().all(|state| {
|
||||
@@ -630,50 +499,25 @@ impl SetDisks {
|
||||
if state.state == DeleteRecoveryState::LegacyAmbiguous {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
settle_outcome(
|
||||
settle(
|
||||
disk,
|
||||
bucket,
|
||||
state,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: context.namespace_owner.as_ref(),
|
||||
budget: context.work_budget.as_deref(),
|
||||
},
|
||||
context.namespace_owner.as_ref(),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if !has_receipt_quorum(&published, write_quorum) {
|
||||
return Err(
|
||||
if published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
} else {
|
||||
DiskError::ErasureWriteQuorum
|
||||
},
|
||||
);
|
||||
if published
|
||||
.iter()
|
||||
.filter(|state| matches!(state, Ok(state) if receipt(state.state)))
|
||||
.count()
|
||||
< write_quorum
|
||||
{
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
}
|
||||
if published.iter().any(|outcome| {
|
||||
matches!(
|
||||
outcome,
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
)
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
let published = published
|
||||
.into_iter()
|
||||
.map(|outcome| match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// Physical cleanup is retryable after the certified decision. An
|
||||
// unavailable participant retains the decision for its eventual return.
|
||||
let ticket = context.recovery_ticket.clone();
|
||||
@@ -826,7 +670,6 @@ impl SetDisks {
|
||||
}
|
||||
let DeleteRecoveryContext {
|
||||
incarnation,
|
||||
work_budget,
|
||||
dry_run,
|
||||
explicit_delete,
|
||||
repair_marker_source,
|
||||
@@ -838,13 +681,7 @@ impl SetDisks {
|
||||
} = context;
|
||||
let disks = self.disk_inventory().await;
|
||||
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| async {
|
||||
match work_budget.as_deref() {
|
||||
Some(budget) => discover_bounded(disk, bucket, object, budget).await,
|
||||
None => discover(disk, bucket, Some(object)).await,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut inspected = 0;
|
||||
let mut offline = !complete_inventory;
|
||||
@@ -875,7 +712,6 @@ impl SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
let mut all_states = BTreeMap::new();
|
||||
let mut registered_evidence_hint = false;
|
||||
for transaction in transactions {
|
||||
let mut complete = Vec::with_capacity(disks.len());
|
||||
let inspections = if let Some((published_tx, published_states)) = published_hint
|
||||
@@ -887,14 +723,8 @@ impl SetDisks {
|
||||
{
|
||||
published_states.to_vec()
|
||||
} else {
|
||||
inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
|
||||
inspect(&disks, bucket, object, transaction).await
|
||||
};
|
||||
if inspections
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
for state in inspections {
|
||||
match state {
|
||||
Ok(state) => {
|
||||
@@ -906,33 +736,6 @@ impl SetDisks {
|
||||
summary.blocked += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
// Candidate UUID directories are not evidence. Only a bound, inspected
|
||||
// journal can create a foreground hint; empty/partial scans rely on the
|
||||
// existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
|
||||
if work_budget.is_some()
|
||||
&& !registered_evidence_hint
|
||||
&& journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
&& journal.volume == bucket
|
||||
&& journal.object == object
|
||||
&& journal.transaction == transaction
|
||||
&& matches!(
|
||||
state.state,
|
||||
DeleteRecoveryState::NewPrepared
|
||||
| DeleteRecoveryState::QuorumCommitted
|
||||
| DeleteRecoveryState::CommitFinished
|
||||
)
|
||||
{
|
||||
self.ctx.delete_recovery_hints().register(
|
||||
crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: self.pool_index,
|
||||
set: self.set_index,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
incarnation,
|
||||
},
|
||||
);
|
||||
registered_evidence_hint = true;
|
||||
}
|
||||
}
|
||||
complete.push(state);
|
||||
}
|
||||
@@ -964,7 +767,7 @@ impl SetDisks {
|
||||
};
|
||||
if let Some((transaction, source)) = prerequisite {
|
||||
Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
|
||||
let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
let refreshed = inspect(&disks, bucket, object, transaction)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1050,7 +853,7 @@ impl SetDisks {
|
||||
if mutated {
|
||||
// A sibling may have restored metadata or reclaimed shared
|
||||
// evidence. Refresh fingerprints before acting on this group.
|
||||
*copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
*copies = inspect(&disks, bucket, object, transaction)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1073,10 +876,7 @@ impl SetDisks {
|
||||
(copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
|
||||
},
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1187,10 +987,7 @@ impl SetDisks {
|
||||
)
|
||||
},
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1204,10 +1001,7 @@ impl SetDisks {
|
||||
.then_some(DeleteRecoveryAction::FinishCommit)
|
||||
},
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1254,10 +1048,7 @@ impl SetDisks {
|
||||
copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1285,7 +1076,7 @@ impl SetDisks {
|
||||
settled.push(copies);
|
||||
} else {
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
@@ -1295,7 +1086,7 @@ impl SetDisks {
|
||||
}
|
||||
for (index, mut copies) in settled.into_iter().enumerate() {
|
||||
if index > 0 {
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction)
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1315,10 +1106,7 @@ impl SetDisks {
|
||||
&mut copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
|
||||
can_commit,
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
namespace_owner.as_ref(),
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1570,459 +1358,6 @@ mod tests {
|
||||
body
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
|
||||
let bucket = "recover-bounded-mixed-quorum";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![73; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let mut outcomes = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let state = disk
|
||||
.inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
|
||||
.await
|
||||
.expect("preparation inspection");
|
||||
let max_entries = match index {
|
||||
0 => 65536,
|
||||
1 | 2 => state.visited,
|
||||
_ => 1,
|
||||
};
|
||||
outcomes.push(
|
||||
disk.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: state.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map(|reply| reply.outcome),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
|
||||
);
|
||||
assert!(matches!(
|
||||
&outcomes[1],
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
));
|
||||
assert!(outcomes[3].is_err());
|
||||
assert!(has_receipt_quorum(&outcomes, 3));
|
||||
assert!(!has_receipt_quorum(&outcomes, 4));
|
||||
outcomes[2] = Err(DiskError::FileAccessDenied);
|
||||
assert!(
|
||||
!has_receipt_quorum(&outcomes, 3),
|
||||
"a write failure cannot substitute for a durable receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
|
||||
use crate::object_api::DeleteRecoveryWorkBudget;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let bucket = "recover-bounded-request";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
for dir in &dirs {
|
||||
let prefix = dir.path().join(bucket).join("prefix");
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for _ in 0..64 {
|
||||
std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("legitimate child");
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("incomplete discovery is not absence");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "prefix".to_owned(),
|
||||
incarnation,
|
||||
}),
|
||||
"an unproven prefix must not move its scan into background recovery"
|
||||
);
|
||||
}
|
||||
for distinct_versions in [false, true] {
|
||||
let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|_| ObjectToDelete {
|
||||
object_name: "prefix".to_owned(),
|
||||
version_id: distinct_versions.then(Uuid::new_v4),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"repeated actions must not each obtain a new allowance: {distinct_versions}"
|
||||
);
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
std::fs::read_dir(dir.path().join(bucket).join("prefix"))
|
||||
.map(|entries| entries.count() == 64)
|
||||
.unwrap_or(false)
|
||||
}));
|
||||
}
|
||||
let error = store
|
||||
.delete_object(bucket, "prefix", ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("ordinary missing key");
|
||||
assert!(
|
||||
crate::error::is_err_object_not_found(&error),
|
||||
"small complete inventory retains no-op semantics: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
let dir = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
dirs.push(dir);
|
||||
}
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("bounded-recovery-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("two pools with two sets");
|
||||
cancel.cancel();
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "recover-bounded-fanout";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
|
||||
let mut keys = BTreeSet::new();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
keys.insert(
|
||||
candidates
|
||||
.iter()
|
||||
.find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
|
||||
.expect("fixture key for set")
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let keys = keys.into_iter().collect::<Vec<_>>();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
assert!(
|
||||
keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
|
||||
"fixture must cover every set"
|
||||
);
|
||||
}
|
||||
}
|
||||
for dir in &dirs {
|
||||
for key in &keys {
|
||||
let prefix = dir.path().join(bucket).join(key);
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for _ in 0..8 {
|
||||
std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("child");
|
||||
}
|
||||
}
|
||||
}
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|index| crate::storage_api_contracts::object::ObjectToDelete {
|
||||
object_name: keys[index % keys.len()].clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"all pools and sets must share the finite allowance"
|
||||
);
|
||||
assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
std::fs::read_dir(dir.path().join(bucket).join(&keys[0]))
|
||||
.map(|entries| entries.count() == 8)
|
||||
.unwrap_or(false)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
|
||||
let bucket = "recover-bounded-fast-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
|
||||
// Inline cleanup needs no directory enumeration and may legitimately
|
||||
// finish with zero scan credits. Require real part files here.
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, "object", "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read non-inline source");
|
||||
assert!(!info.inline_data(), "the fixture must require physical part cleanup");
|
||||
let data_dir = info.data_dir.expect("physical data directory");
|
||||
let part = std::path::PathBuf::from(disk.endpoint().to_string())
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
assert!(part.is_file(), "the cleanup scan must encounter a real part");
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a cached prepared receipt commits independently of cleanup budget");
|
||||
assert!(
|
||||
store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation")
|
||||
),
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
!disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("retained recovery proof")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background finishes committed cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
|
||||
let bucket = "recover-bounded-candidate";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
let transaction = Uuid::new_v4();
|
||||
for directory in &dirs {
|
||||
let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
|
||||
std::fs::create_dir_all(&candidate).expect("candidate path");
|
||||
std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.delete_object(bucket, "candidate", ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "candidate".to_owned(),
|
||||
incarnation,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
|
||||
let bucket = "recover-bounded-evidence";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
let payload = vec![71; 4096];
|
||||
put(&store, bucket, &payload, false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let before = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("before refusal");
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("no incomplete recovery admission");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
let after = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("after refusal");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background-compatible recovery");
|
||||
assert_eq!(body(&store, bucket, None).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
|
||||
let bucket = "recover-bounded-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![72; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: None,
|
||||
};
|
||||
let ticket = set.ctx.delete_recovery_hints().register(key.clone());
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
|
||||
let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
// This hook disables local fast cleanup and runs only after receipt
|
||||
// publication and the shared recovery path's key-wide GC inspection.
|
||||
// Exhausting preparation instead would correctly reject the delete.
|
||||
test_hooks::before_gc_finalize(bucket, "object", {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Arc::new(move || {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Box::pin(async move {
|
||||
assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
|
||||
exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
|
||||
})
|
||||
})
|
||||
});
|
||||
set.publish_delete_commit(
|
||||
bucket,
|
||||
"object",
|
||||
transaction,
|
||||
&set.disk_inventory().await,
|
||||
set.set_drive_count / 2 + 1,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: Some(Arc::clone(&budget)),
|
||||
recovery_ticket: ticket,
|
||||
..DeleteRecoveryContext::new(None, &|| true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the durable decision still succeeds");
|
||||
assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
|
||||
assert_eq!(budget.remaining(), 0);
|
||||
assert!(set.ctx.delete_recovery_hints().is_pending(&key));
|
||||
let state = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("retained decision");
|
||||
assert!(receipt(state.state));
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("eventual cleanup");
|
||||
assert!(
|
||||
disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("cleaned proofs")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
|
||||
let bucket = "recover-undecided";
|
||||
|
||||
@@ -34,6 +34,7 @@ mod heal_lock_wait_tests;
|
||||
#[derive(Clone, Copy)]
|
||||
struct ExplicitVersionHeal<'a> {
|
||||
opts: &'a HealOpts,
|
||||
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
|
||||
allow_regeneration: bool,
|
||||
preserve_unrepairable: bool,
|
||||
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
|
||||
@@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> {
|
||||
retirement: Option<&'a MarkerRetirementContext<'a>>,
|
||||
}
|
||||
|
||||
async fn restore_heal_exclusive_guard(
|
||||
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> disk::error::Result<()> {
|
||||
tokio::time::timeout(get_lock_acquire_timeout(), async {
|
||||
loop {
|
||||
{
|
||||
let mut guard = guard.lock().await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// A writer may have waited on a peer that upgraded while
|
||||
// another peer was still waiting for its readers to drain.
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
return if matches!(waiting, Ok(Some(false))) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::Timeout)
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let waiting =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
|
||||
drop(tokio::spawn(async move {
|
||||
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}));
|
||||
}
|
||||
|
||||
// Keep the large heal state machine and its construction temporary out of
|
||||
// each dispatch caller's poll frame. Ordinary object writes do not use this
|
||||
// heal-only allocation boundary.
|
||||
@@ -844,6 +897,7 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: None,
|
||||
@@ -994,6 +1048,7 @@ impl SetDisks {
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
let ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard,
|
||||
allow_regeneration: allow_explicit_version_regen,
|
||||
preserve_unrepairable,
|
||||
commit_fence,
|
||||
@@ -1701,7 +1756,29 @@ impl SetDisks {
|
||||
let dst_data_dir = data_dir;
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = async {
|
||||
let mut shared_staging = false;
|
||||
if let Some(guard) = staging_guard {
|
||||
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.supports_heal_lock_transition().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if supported {
|
||||
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.downgrade_committed_tail().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if converted {
|
||||
shared_staging = true;
|
||||
} else {
|
||||
// Some peers may have converted. Abort without committing;
|
||||
// dropping the guard releases each retained lease.
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stage = async {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
|
||||
|
||||
@@ -1926,17 +2003,43 @@ impl SetDisks {
|
||||
|
||||
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
|
||||
Ok(None)
|
||||
}
|
||||
.await;
|
||||
};
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
|
||||
if let Some(guard) = staging_guard {
|
||||
tokio::select! {
|
||||
result = stage => result,
|
||||
_ = wait_for_heal_writer(guard) => {
|
||||
// The UUID prefix is private to this attempt. Cleanup must not
|
||||
// retain the namespace lock while a foreground writer waits;
|
||||
// the stale-tmp sweeper also covers a cancelled disk operation.
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
};
|
||||
match stage_result {
|
||||
Ok(None) => {}
|
||||
failure => {
|
||||
// Release readers and writers before removing all parts
|
||||
// staged under this heal's exclusively owned prefix.
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
if shared_staging {
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
} else {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}
|
||||
return failure.map(|error| (result, error));
|
||||
}
|
||||
}
|
||||
if shared_staging
|
||||
&& let Some(guard) = staging_guard
|
||||
&& let Err(error) = restore_heal_exclusive_guard(guard).await
|
||||
{
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
|
||||
if _read_repair_read_lock_guard
|
||||
@@ -3616,6 +3719,7 @@ impl SetDisks {
|
||||
&version,
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(can_commit),
|
||||
@@ -3676,6 +3780,7 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &read_opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -3702,7 +3807,7 @@ impl SetDisks {
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
|
||||
Some(Arc::new(
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
measure_heal_object_lock(
|
||||
"heal_object",
|
||||
HealObjectLockKind::Write,
|
||||
@@ -3714,7 +3819,7 @@ impl SetDisks {
|
||||
.narrow_to_disk()
|
||||
.unwrap_or_else(DiskError::other)
|
||||
})?,
|
||||
))
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -3732,10 +3837,18 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
|
||||
guard.lock().await.lock_lost_signal()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
// Recovery cleanup can outlive its future. Never permit reads until
|
||||
// every physical operation holding its recovery owner has finished.
|
||||
let recovery_tail = Arc::new(());
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3762,6 +3875,7 @@ impl SetDisks {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
Arc::clone(&recovery_tail),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -3885,6 +3999,9 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -3909,6 +4026,9 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -6047,6 +6167,7 @@ mod heal_result_report_tests {
|
||||
"",
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -10637,6 +10758,369 @@ mod heal_result_report_tests {
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-storage-entry";
|
||||
let object = "object.bin";
|
||||
let expected = vec![0x37; 1024 * 1024];
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external object data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing).await.expect("remove shard");
|
||||
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
let result = heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair: false,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, absence)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("read-repair reached shard reconstruction");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
set.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("background heal reconstruction should allow concurrent readers")
|
||||
.expect("open object while reconstruction is paused");
|
||||
let mut during_heal = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
|
||||
.await
|
||||
.expect("read completes while reconstruction is paused")
|
||||
.expect("read object during reconstruction");
|
||||
assert_eq!(during_heal, expected);
|
||||
drop(reader);
|
||||
release.notify_one();
|
||||
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("repair finishes")
|
||||
.expect("repair task joins");
|
||||
let (_, error) = result.expect("repair operation succeeds");
|
||||
assert!(error.is_none(), "repair failed: {error:?}");
|
||||
assert!(absence.is_none());
|
||||
assert!(missing.exists(), "missing shard was not restored");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-long-put";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let put = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes after releasing reconstruction")
|
||||
.expect("heal task joins");
|
||||
put.expect("foreground PUT finishes during reconstruction")
|
||||
.expect("foreground PUT must not time out");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
drop(_pause);
|
||||
let mut absence = None;
|
||||
let (_, retry_error) = set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a later heal must retry the current object");
|
||||
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read foreground write after heal");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-version-delete";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let original = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original version");
|
||||
let version = original.version_id.expect("version id").to_string();
|
||||
let survivor = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x52; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write surviving version")
|
||||
.version_id
|
||||
.expect("surviving version id");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("read original metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal_version = version.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
&heal_version,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut None,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let delete = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes")
|
||||
.expect("heal task joins");
|
||||
delete
|
||||
.expect("foreground delete finishes")
|
||||
.expect("foreground delete succeeds");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
let result = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
|
||||
"stale heal must not resurrect the deleted version: {result:?}"
|
||||
);
|
||||
let mut survivor = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(survivor.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version remains readable");
|
||||
let mut body = Vec::new();
|
||||
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
|
||||
assert_eq!(body, vec![0x52; 1024]);
|
||||
for disk in &disks {
|
||||
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
|
||||
assert!(
|
||||
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
|
||||
"deleted version must remain absent on each disk: {physical:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -95,8 +95,13 @@ impl SetDisks {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resource = ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
);
|
||||
let guard = self
|
||||
.namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
|
||||
.namespace_lock_for_key(resource)
|
||||
.await
|
||||
.get_write_lock(super::super::get_lock_acquire_timeout())
|
||||
.await
|
||||
@@ -106,94 +111,6 @@ impl SetDisks {
|
||||
Ok(Arc::new(guard))
|
||||
}
|
||||
|
||||
fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
|
||||
ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
)
|
||||
}
|
||||
|
||||
/// Batched write-lock acquisition for the whole key set of one bulk
|
||||
/// mutation: one lock round instead of one quorum round-trip per key.
|
||||
///
|
||||
/// Bulk delete is the only multi-key write-lock holder. The batched round
|
||||
/// is deadlock-free by construction (a single `lock_batch` RPC per lock
|
||||
/// client, and every lock server orders concurrent batches by the same
|
||||
/// global (shard, key) order), so unlike an optimistic probe wave it
|
||||
/// needs no canonical serial fallback. Outcomes are per key and use the
|
||||
/// same error shape as the single-key paths.
|
||||
async fn namespace_write_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
resources: &[ObjectKey],
|
||||
timeout: std::time::Duration,
|
||||
mode: &'static str,
|
||||
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
|
||||
if resources.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let lock = self.namespace_lock_for_key(resources[0].clone()).await;
|
||||
lock.get_write_locks_batch(resources, timeout)
|
||||
.await
|
||||
.into_iter()
|
||||
.zip(resources)
|
||||
.map(|(result, resource)| {
|
||||
result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Batched raw namespace write locks for a bulk delete's key set. The
|
||||
/// resource keys use the same disk-identity folding as `new_ns_lock`, so
|
||||
/// aliases (`a\b` vs `a/b`) still share one lock.
|
||||
pub(crate) async fn acquire_object_write_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: &[&str],
|
||||
timeout: std::time::Duration,
|
||||
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resources: Vec<ObjectKey> = objects
|
||||
.iter()
|
||||
.map(|object| ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object_name_disk_identity(object).as_ref()),
|
||||
version: None,
|
||||
})
|
||||
.collect();
|
||||
self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
|
||||
}
|
||||
|
||||
/// Batched delete-transaction locks for a bulk delete's key set; see
|
||||
/// [`Self::acquire_delete_transaction_lock`] for the resource shape.
|
||||
pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: &[&str],
|
||||
timeout: std::time::Duration,
|
||||
) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
|
||||
let resources: Vec<ObjectKey> = objects
|
||||
.iter()
|
||||
.map(|object| self.delete_transaction_resource(bucket, object))
|
||||
.collect();
|
||||
let mut guards = Vec::with_capacity(resources.len());
|
||||
for (object, result) in objects.iter().zip(
|
||||
self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
|
||||
.await,
|
||||
) {
|
||||
match result {
|
||||
Ok(guard) => {
|
||||
#[cfg(test)]
|
||||
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
|
||||
let _ = object;
|
||||
guards.push(Ok(Arc::new(guard)));
|
||||
}
|
||||
Err(error) => guards.push(Err(error)),
|
||||
}
|
||||
}
|
||||
guards
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||
match err {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -165,6 +165,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock,
|
||||
false,
|
||||
DecommissionCapacityAdmission::Mutation,
|
||||
)
|
||||
.await?
|
||||
@@ -275,6 +276,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock,
|
||||
false,
|
||||
DecommissionCapacityAdmission::Mutation,
|
||||
)
|
||||
.await?
|
||||
|
||||
@@ -52,8 +52,8 @@ pub(crate) mod list {
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
|
||||
MultipartUploadInspection, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::{
|
||||
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState},
|
||||
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState, table_bucket_catalog_metadata_prefix},
|
||||
utils::is_meta_bucketname,
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
@@ -145,41 +145,11 @@ async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
|
||||
use crate::bucket::metadata::*;
|
||||
|
||||
let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
|
||||
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
|
||||
if bucket != "ahm-replacement" {
|
||||
prefixes.push(root);
|
||||
return prefixes;
|
||||
}
|
||||
|
||||
// This valid user bucket shares its metadata directory with raw drive
|
||||
// replacement intents, seals and recovery proofs. Delete only bucket-owned
|
||||
// paths, including legacy standalone configuration, never the shared root.
|
||||
// S3 objects live in the separate user volume; configuration APIs write the
|
||||
// aggregate metadata file, so clients cannot populate arbitrary siblings.
|
||||
prefixes.extend(
|
||||
[
|
||||
BUCKET_METADATA_FILE,
|
||||
BUCKET_INCARNATION_FILE,
|
||||
BUCKET_POLICY_CONFIG,
|
||||
BUCKET_QUOTA_CONFIG_FILE,
|
||||
BUCKET_TARGETS_FILE,
|
||||
BUCKET_ACL_CONFIG,
|
||||
BUCKET_TABLE_CONFIG,
|
||||
BUCKET_DURABILITY_CONFIG,
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG,
|
||||
".replication",
|
||||
"integrity-jobs",
|
||||
crate::data_usage::DATA_USAGE_CACHE_NAME,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(XML_BUCKET_CONFIG_FILES)
|
||||
.map(|file| format!("{root}/{file}")),
|
||||
);
|
||||
prefixes
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
|
||||
[
|
||||
table_bucket_catalog_metadata_prefix(bucket),
|
||||
format!("{BUCKET_META_PREFIX}/{bucket}"),
|
||||
]
|
||||
}
|
||||
|
||||
fn bucket_deleted_marker_prefix(bucket: &str) -> String {
|
||||
@@ -190,48 +160,41 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
|
||||
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
|
||||
}
|
||||
|
||||
// Box the child before constructing either lease wrapper. Keeping F inline in
|
||||
// nested select futures multiplies poll-frame temporaries along DELETE -> quota
|
||||
// cleanup -> metadata GET. This boundary must not inline back into that chain.
|
||||
#[inline(never)]
|
||||
pub(crate) fn await_bucket_namespace_operation<'a, T: 'a, F>(
|
||||
guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &'a str,
|
||||
pub(crate) async fn await_bucket_namespace_operation<T, F>(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
future: F,
|
||||
) -> impl Future<Output = Result<T>> + 'a
|
||||
) -> Result<T>
|
||||
where
|
||||
F: Future<Output = Result<T>> + 'a,
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
let future = Box::pin(future);
|
||||
async move {
|
||||
let Some(guard) = guard else {
|
||||
return future.await;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost before {operation}: {bucket}"
|
||||
)));
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost during {operation}: {bucket}"
|
||||
))),
|
||||
result = future => result,
|
||||
}
|
||||
let Some(guard) = guard else {
|
||||
return future.await;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost before {operation}: {bucket}"
|
||||
)));
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost during {operation}: {bucket}"
|
||||
))),
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
|
||||
fn await_bucket_lifecycle_operation<'a, T: 'a, F>(
|
||||
lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
namespace_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &'a str,
|
||||
async fn await_bucket_lifecycle_operation<T, F>(
|
||||
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
operation: &'static str,
|
||||
future: F,
|
||||
) -> impl Future<Output = Result<T>> + 'a
|
||||
) -> Result<T>
|
||||
where
|
||||
F: Future<Output = Result<T>> + 'a,
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
await_bucket_namespace_operation(
|
||||
lifecycle_guard,
|
||||
@@ -239,6 +202,7 @@ where
|
||||
operation,
|
||||
await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
|
||||
@@ -514,18 +478,10 @@ impl ECStore {
|
||||
let mut tags = HashMap::with_capacity(tagging.tag_set.len());
|
||||
for tag in &tagging.tag_set {
|
||||
let (Some(key), Some(value)) = (&tag.key, &tag.value) else {
|
||||
return Err(crate::bucket::metadata::unreadable_config_error(
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
metadata.tagging_config_xml.len(),
|
||||
));
|
||||
return Err(Error::other("stored bucket tag is missing a key or value"));
|
||||
};
|
||||
if key.is_empty() || tags.insert(key.clone(), value.clone()).is_some() {
|
||||
return Err(crate::bucket::metadata::unreadable_config_error(
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
metadata.tagging_config_xml.len(),
|
||||
));
|
||||
return Err(Error::other("stored bucket tags contain empty or duplicate keys"));
|
||||
}
|
||||
}
|
||||
Ok(tags)
|
||||
@@ -1717,17 +1673,7 @@ impl ECStore {
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket retirement publication",
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
|
||||
)
|
||||
.await?;
|
||||
// The lifecycle fence excludes live reservation owners. A crash-left
|
||||
// V1 ledger must not block quota admission after same-name recreation.
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"retired bucket quota ledger cleanup",
|
||||
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
|
||||
crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1878,20 +1824,6 @@ mod tests {
|
||||
assert!(!xlmeta.diagnostic_truncated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_namespace_operation_keeps_nested_future_state_off_the_poll_stack() {
|
||||
let payload = std::hint::black_box([7u8; 128 * 1024]);
|
||||
let operation = super::await_bucket_lifecycle_operation(None, None, "bucket", "large metadata operation", async move {
|
||||
tokio::task::yield_now().await;
|
||||
Ok(std::hint::black_box(payload)[0])
|
||||
});
|
||||
assert!(
|
||||
std::mem::size_of_val(&operation) < 1024,
|
||||
"nested lease wrappers must not retain the concrete storage future inline"
|
||||
);
|
||||
assert_eq!(operation.await.expect("the boxed operation must still execute"), 7);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
|
||||
let ttl = Duration::from_millis(20);
|
||||
@@ -2308,13 +2240,7 @@ mod tests {
|
||||
let result = store.get_bucket_tags_for_policy(bucket).await;
|
||||
match expected {
|
||||
Some(tags) => assert_eq!(&result.expect("valid tag configuration"), tags),
|
||||
None => assert!(
|
||||
crate::bucket::metadata::unreadable_config_refusal(
|
||||
&result.expect_err("corrupt tags must not become an untagged bucket")
|
||||
)
|
||||
.is_some_and(|failure| failure.config_file == super::BUCKET_TAGGING_CONFIG),
|
||||
"corrupt tags must carry a typed tagging refusal: {xml:?}"
|
||||
),
|
||||
None => assert!(result.is_err(), "corrupt tags must not become an untagged bucket: {xml:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2368,8 +2294,17 @@ mod tests {
|
||||
drop(
|
||||
lock.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("fixture rename tail should finish before checking its generation"),
|
||||
.expect("fixture object lock should be available"),
|
||||
);
|
||||
// A quorum-acknowledged rename can release the object lock while its
|
||||
// physical tail still holds the instance-wide namespace owner.
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while ecstore.ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("fixture namespace owners should drain before checking its generation");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_put.saturating_add(3),
|
||||
@@ -2516,187 +2451,6 @@ mod tests {
|
||||
assert!(prefixes.contains(&"buckets/analytics".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replacement_named_bucket_delete_preserves_recovery_and_recreates_clean_metadata() {
|
||||
use crate::bucket::metadata::{BUCKET_QUOTA_CONFIG_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
|
||||
let (_temp, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "ahm-replacement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create legal bucket");
|
||||
let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
|
||||
for (config, bytes) in [
|
||||
(BUCKET_QUOTA_CONFIG_FILE, br#"{"quota":1024,"quota_type":"Hard"}"#.as_slice()),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
b"<Tagging><TagSet><Tag><Key>old</Key><Value>value</Value></Tag></TagSet></Tagging>".as_slice(),
|
||||
),
|
||||
(
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".as_slice(),
|
||||
),
|
||||
] {
|
||||
metadata_sys::update_in(&store.ctx, bucket, config, bytes.to_vec())
|
||||
.await
|
||||
.expect("persist old configuration");
|
||||
}
|
||||
let ledger_path = format!("config/quota-ledger/{bucket}.json");
|
||||
// V1 precharges outstanding growth: four live bytes plus the
|
||||
// crash-left reservation's 100 bytes must already be accounted.
|
||||
let ledger = serde_json::to_vec(&serde_json::json!({
|
||||
"version": 1, "bucket_incarnation": old_incarnation,
|
||||
"quota_revision_unix_nanos": 0, "accounted_usage": 104,
|
||||
"reservations": {Uuid::new_v4().to_string(): {
|
||||
"object": "crashed-put", "old_size": 0, "new_size": 100,
|
||||
"created_at": 0, "pool_index": 0, "set_index": 0, "commit_started": false
|
||||
}}, "reconcile_required": true
|
||||
}))
|
||||
.expect("encode crash-left ledger");
|
||||
|
||||
// The storage API requires the same legacy-quota admission supplied
|
||||
// by the S3 handler, even for this small fixture object.
|
||||
let mut live_options = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(live_options.set_quota_admission(0, 1024));
|
||||
// Failed deletion must leave both the live incarnation and its ledger alone.
|
||||
store
|
||||
.put_object(bucket, "live", &mut PutObjReader::from_vec(b"live".to_vec()), &live_options)
|
||||
.await
|
||||
.expect("write live object");
|
||||
crate::config::com::save_config(store.clone(), &ledger_path, ledger.clone())
|
||||
.await
|
||||
.expect("persist old ledger");
|
||||
assert!(matches!(
|
||||
store.delete_bucket(bucket, &DeleteBucketOptions::default()).await,
|
||||
Err(StorageError::BucketNotEmpty(_))
|
||||
));
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(store.clone(), &ledger_path)
|
||||
.await
|
||||
.expect("live ledger remains"),
|
||||
ledger
|
||||
);
|
||||
// Remove every version; the configured bucket is otherwise still empty.
|
||||
let versions = store
|
||||
.clone()
|
||||
.list_object_versions(bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list live versions");
|
||||
for object in versions.objects {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
&object.name,
|
||||
ObjectOptions {
|
||||
version_id: object.version_id.map(|id| id.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove live version");
|
||||
}
|
||||
|
||||
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
|
||||
let task = Uuid::new_v4();
|
||||
let recovery_names = [
|
||||
"ahm_replacement_intent.json",
|
||||
"ahm_replacement_intent_seal",
|
||||
"ahm_replacement_completion_proof.json",
|
||||
]
|
||||
.map(|suffix| format!("{task}_{suffix}"));
|
||||
let disk_paths = store.pools[0].disk_set[0]
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|disk| PathBuf::from(disk.endpoint().to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disk_paths {
|
||||
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
|
||||
for name in &recovery_names {
|
||||
tokio::fs::write(directory.join(name), b"durable heal record")
|
||||
.await
|
||||
.expect("seed raw recovery record");
|
||||
}
|
||||
// Legacy standalone settings and bucket-owned subtrees must not survive.
|
||||
for name in [
|
||||
"quota.json",
|
||||
"versioning.xml",
|
||||
"policy.json",
|
||||
"bucket-targets.json",
|
||||
"integrity-jobs/job.json",
|
||||
".replication/resync.bin",
|
||||
".usage-cache.bin",
|
||||
] {
|
||||
let path = directory.join(name);
|
||||
tokio::fs::create_dir_all(path.parent().expect("fixture parent"))
|
||||
.await
|
||||
.expect("legacy parent");
|
||||
tokio::fs::write(path, b"old bucket configuration")
|
||||
.await
|
||||
.expect("seed legacy configuration");
|
||||
}
|
||||
}
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete legal same-name bucket");
|
||||
assert!(matches!(
|
||||
crate::config::com::read_config(store.clone(), &ledger_path).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
for disk in &disk_paths {
|
||||
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
|
||||
for name in &recovery_names {
|
||||
assert_eq!(
|
||||
tokio::fs::read(directory.join(name)).await.expect("recovery record survives"),
|
||||
b"durable heal record"
|
||||
);
|
||||
}
|
||||
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket)
|
||||
.into_iter()
|
||||
.filter(|p| p.starts_with(&root))
|
||||
{
|
||||
assert!(
|
||||
!disk.join(RUSTFS_META_BUCKET).join(prefix).exists(),
|
||||
"bucket-owned metadata must be removed"
|
||||
);
|
||||
}
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate legal bucket");
|
||||
let fresh = metadata_sys::get_in(&store.ctx, bucket).await.expect("fresh metadata");
|
||||
assert_ne!(fresh.bucket_incarnation_id, old_incarnation);
|
||||
assert!(fresh.quota_config_json.is_empty());
|
||||
assert!(fresh.tagging_config_xml.is_empty());
|
||||
assert!(fresh.versioning_config_xml.is_empty());
|
||||
assert!(fresh.policy_config_json.is_empty());
|
||||
// The strict purge/recovery cleanup must share the same ownership boundary.
|
||||
store
|
||||
.cleanup_deleted_bucket_metadata_strict(bucket, true)
|
||||
.await
|
||||
.expect("strict metadata cleanup");
|
||||
for disk in &disk_paths {
|
||||
for name in &recovery_names {
|
||||
assert_eq!(
|
||||
tokio::fs::read(disk.join(RUSTFS_META_BUCKET).join(&root).join(name))
|
||||
.await
|
||||
.expect("strict cleanup preserves recovery"),
|
||||
b"durable heal record"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
|
||||
assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
use crate::error::{Result, StorageError};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use rustfs_lock::distributed_lock::LockLostSignal;
|
||||
@@ -65,7 +66,7 @@ struct RegisteredGuard {
|
||||
|
||||
enum LockLossProbe {
|
||||
Distributed(Arc<LockLostSignal>),
|
||||
Local,
|
||||
Local(Option<rustfs_lock::fast_lock::guard::FastLockGuardProbe>),
|
||||
#[cfg(test)]
|
||||
Test(Arc<std::sync::atomic::AtomicBool>),
|
||||
}
|
||||
@@ -74,14 +75,19 @@ impl LockLossProbe {
|
||||
fn from_guard(guard: &NamespaceLockGuard) -> Self {
|
||||
match guard.lock_lost_signal() {
|
||||
Some(signal) => Self::Distributed(signal),
|
||||
None => Self::Local,
|
||||
None => Self::Local(match guard {
|
||||
NamespaceLockGuard::Fast(guard) => guard.release_probe(),
|
||||
NamespaceLockGuard::Standard(_) => None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lost(&self) -> bool {
|
||||
// Registry callers hold entries before checking a local shard; shard
|
||||
// operations never acquire the bucket fence registry.
|
||||
match self {
|
||||
Self::Distributed(signal) => signal.is_lost(),
|
||||
Self::Local => false,
|
||||
Self::Local(probe) => probe.as_ref().is_some_and(|probe| probe.is_lost()),
|
||||
#[cfg(test)]
|
||||
Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst),
|
||||
}
|
||||
@@ -98,9 +104,74 @@ pub(super) struct FenceRegistration {
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BucketFenceRegistry {
|
||||
entries: Mutex<HashMap<String, FenceEntry>>,
|
||||
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ReservationFenceSlot {
|
||||
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
|
||||
}
|
||||
|
||||
/// Shared by short bursts of reservations, but never kept alive by the cache.
|
||||
pub(super) struct ReservationFenceGuard {
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
created_at: tokio::time::Instant,
|
||||
_slot: Arc<ReservationFenceSlot>,
|
||||
}
|
||||
|
||||
impl ReservationFenceGuard {
|
||||
pub(super) fn new(
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
slot: Arc<ReservationFenceSlot>,
|
||||
) -> Self {
|
||||
Self {
|
||||
publication,
|
||||
lifecycle,
|
||||
created_at: tokio::time::Instant::now(),
|
||||
_slot: slot,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_reusable(&self) -> bool {
|
||||
// Stop admitting new reservations through this shared reader so a
|
||||
// queued bucket writer can progress after the existing tails drain.
|
||||
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
|
||||
}
|
||||
|
||||
fn is_held(&self) -> bool {
|
||||
!self.publication.is_lock_lost()
|
||||
&& !self.publication.is_released()
|
||||
&& !self.lifecycle.is_lock_lost()
|
||||
&& self
|
||||
.lifecycle
|
||||
.namespace_lock_guard()
|
||||
.is_some_and(|guard| !guard.is_released())
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
|
||||
fn is_valid(&self) -> bool {
|
||||
self.is_held()
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketFenceRegistry {
|
||||
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
|
||||
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
|
||||
let key = (bucket.to_owned(), incarnation);
|
||||
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
|
||||
return slot;
|
||||
}
|
||||
if slots.len() >= 1024 {
|
||||
slots.retain(|_, slot| slot.strong_count() != 0);
|
||||
}
|
||||
let slot = Arc::new(ReservationFenceSlot::default());
|
||||
slots.insert(key, Arc::downgrade(&slot));
|
||||
slot
|
||||
}
|
||||
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
|
||||
@@ -234,7 +305,9 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
self.inner
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
|
||||
@@ -352,6 +425,67 @@ mod tests {
|
||||
reg.exit("b", fourth.token, false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_local_lock_does_not_add_a_namespace_fence() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Disabled(
|
||||
rustfs_lock::fast_lock::DisabledLockManager::new(),
|
||||
));
|
||||
let lock = NamespaceLock::with_local_manager("disabled-fence".into(), manager);
|
||||
let guard = lock
|
||||
.get_read_lock(ObjectKey::new("b", "object"), "owner", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("disabled lock");
|
||||
let mut opts = crate::object_api::ObjectOptions::default();
|
||||
opts.add_namespace_lock_guard(&guard);
|
||||
assert!(opts.namespace_lock_fence.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_local_fence_loss_invalidates_live_incarnation_memo() {
|
||||
let manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
|
||||
let global = Arc::new(rustfs_lock::GlobalLockManager::Enabled(manager.clone()));
|
||||
let lock = NamespaceLock::with_local_manager("forced-fence".into(), global);
|
||||
let key = ObjectKey::new("b", "lifecycle");
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
let inner = lock
|
||||
.get_read_lock(key.clone(), "first", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("first lock");
|
||||
let mut first = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
|
||||
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
|
||||
let mut opts = crate::object_api::ObjectOptions::default();
|
||||
first.attach_to_object_options(&mut opts);
|
||||
opts.add_namespace_lock_guard(first.namespace_lock_guard().expect("local guard"));
|
||||
assert!(
|
||||
!opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("lifecycle fence")
|
||||
.is_lock_lost()
|
||||
);
|
||||
assert!(!opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
assert!(
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("attached fence")
|
||||
.is_lock_lost()
|
||||
);
|
||||
assert!(opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
|
||||
let writer = lock
|
||||
.get_write_lock(key.clone(), "writer", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("writer");
|
||||
drop(writer);
|
||||
let inner = lock
|
||||
.get_read_lock(key, "second", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("second lock");
|
||||
let mut second = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
|
||||
assert_eq!(second.validate(true, || async { Ok(uuid(8)) }).await.expect("reload"), uuid(8));
|
||||
assert!(first.validate(true, || async { Ok(uuid(7)) }).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fence_guard_forwards_distributed_lock_loss_to_registry() {
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
|
||||
@@ -368,7 +368,7 @@ impl ECStore {
|
||||
Some(expected) => expected,
|
||||
None => self.bucket_incarnation_id_from_disk(bucket).await?,
|
||||
};
|
||||
let object = object.map(encode_dir_object);
|
||||
let object = object.map(str::to_owned);
|
||||
let prefix = prefix.map(str::to_owned);
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
|
||||
@@ -430,7 +430,7 @@ impl ECStore {
|
||||
HealItemType::Bucket
|
||||
}
|
||||
.to_string(),
|
||||
object: object.map(|object| decode_dir_object(&object)).or(prefix).unwrap_or_default(),
|
||||
object: object.or(prefix).unwrap_or_default(),
|
||||
bucket,
|
||||
detail: format!(
|
||||
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
|
||||
@@ -656,48 +656,12 @@ impl ECStore {
|
||||
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
/// Read the published bucket identity without fabricating or migrating metadata.
|
||||
/// Absence is distinct from quorum, corruption and transaction-fence failures.
|
||||
pub async fn replacement_bucket_incarnation_from_disk(self: &Arc<Self>, bucket: &str) -> Result<Option<uuid::Uuid>> {
|
||||
let transaction =
|
||||
crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let incarnation = crate::bucket::metadata::load_bucket_incarnation(self.clone(), bucket).await?;
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(incarnation)
|
||||
}
|
||||
|
||||
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
|
||||
pub async fn heal_replacement_bucket_metadata(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
// Reject an invalid request before touching the drives so callers keep
|
||||
// the PreconditionFailed contract for bad buckets and options.
|
||||
if opts.pool.is_none()
|
||||
|| opts.set.is_none()
|
||||
|| targets.is_empty()
|
||||
|| opts.dry_run
|
||||
|| opts.no_lock
|
||||
|| check_valid_bucket_name_strict(bucket).is_err()
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let expected = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
self.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Keep a replacement pass bound to its admitted bucket through metadata commit.
|
||||
pub async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
|
||||
@@ -734,8 +698,9 @@ impl ECStore {
|
||||
|
||||
// Read the persisted identity without lazy migration: a replacement
|
||||
// must never turn lost configuration into newly fabricated defaults.
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let targets = targets.to_vec();
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, opts| async move {
|
||||
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
|
||||
// Match config writers: lifecycle, metadata transaction, object locks.
|
||||
// Keep the transaction stable through target readback, including when
|
||||
// the caller is cancelled and the storage owner finishes its write.
|
||||
@@ -1057,7 +1022,6 @@ impl ECStore {
|
||||
set_index: usize,
|
||||
targets: &[String],
|
||||
) -> Result<bool> {
|
||||
let object = encode_dir_object(object);
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -1070,7 +1034,7 @@ impl ECStore {
|
||||
)
|
||||
})?;
|
||||
|
||||
set.replacement_targets_have_version(bucket, &object, version_id, targets)
|
||||
set.replacement_targets_have_version(bucket, object, version_id, targets)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
@@ -2895,105 +2859,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_directory_object_readback_and_exact_recovery_use_encoded_key() {
|
||||
let (_root, store, shutdown) = heal_store_with_pool_count(1).await;
|
||||
let bucket = "directory-replacement-readback";
|
||||
let object = "folder/";
|
||||
let encoded = encode_dir_object(object);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(Vec::new()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write directory object through the public store entry");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let targets = vec![set.set_endpoints[0].to_string()];
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("readback")
|
||||
);
|
||||
let disk = set.disks.read().await[0].clone().expect("target disk");
|
||||
disk.delete(
|
||||
bucket,
|
||||
&encoded,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove target directory shard");
|
||||
assert!(
|
||||
!store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("missing target")
|
||||
);
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
let (result, error) = store
|
||||
.heal_object(bucket, object, "", &opts)
|
||||
.await
|
||||
.expect("heal directory object");
|
||||
assert!(error.is_none(), "heal failed: {error:?}");
|
||||
assert_eq!(result.object, object);
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("healed readback")
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, &encoded, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("encoded readback")
|
||||
);
|
||||
|
||||
// Exact-object recovery must inspect the same physical namespace and
|
||||
// reject corrupt proof rather than silently report no transactions.
|
||||
let metadata = disk
|
||||
.read_all(bucket, &format!("{encoded}/xl.meta"))
|
||||
.await
|
||||
.expect("healed metadata");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{encoded}/{}/delete-prepared", Uuid::new_v4()),
|
||||
b"invalid proof".to_vec().into(),
|
||||
)
|
||||
.await
|
||||
.expect("seed corrupt recovery proof");
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some(object), None, &HealOpts { dry_run: true, ..opts })
|
||||
.await
|
||||
.expect_err("directory recovery must discover and reject the corrupt proof");
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, &format!("{encoded}/xl.meta"))
|
||||
.await
|
||||
.expect("unchanged metadata"),
|
||||
metadata
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
@@ -3827,177 +3692,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// The RPC lock service uses the raw ObjectKey and one backend per node,
|
||||
// even when the requesting sets carry different pool namespace strings.
|
||||
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
|
||||
use rustfs_lock::client::{LockClient, local::LocalClient};
|
||||
|
||||
let clients = || {
|
||||
(0..2)
|
||||
.map(|_| {
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let first = clients();
|
||||
let second = if shared { first.clone() } else { clients() };
|
||||
let mut pools = source.pools.clone();
|
||||
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
|
||||
for set in &mut Arc::make_mut(pool).disk_set {
|
||||
Arc::make_mut(set).lockers = lockers.clone();
|
||||
}
|
||||
}
|
||||
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
||||
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
|
||||
Arc::new(ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: Default::default(),
|
||||
pools,
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(state),
|
||||
ctx: Arc::clone(&source.ctx),
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
|
||||
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let first_lock = first
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("first pool lock");
|
||||
let second_lock = second
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("second pool lock");
|
||||
let guard = first_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"different pool namespaces still conflict on the same backend and object key"
|
||||
);
|
||||
drop(guard);
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(3))
|
||||
.await
|
||||
.expect("second writer should acquire after the first releases");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let missing = remove_pool_meta_shard(&store, 1).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cross-pool repair must not reenter the shared writer")
|
||||
.expect("pool metadata repair should return a result");
|
||||
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
assert!(
|
||||
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
|
||||
"the selected missing shard must be restored"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
|
||||
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let lock = target
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("target lock");
|
||||
let guard = lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("hold target lock");
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let heal_store = Arc::clone(&store);
|
||||
let mut heal = tokio::spawn(async move {
|
||||
heal_store
|
||||
.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
barrier.wait_until_external_heal_target_lock_attempted(),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must attempt the independent target lock");
|
||||
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
|
||||
drop(guard);
|
||||
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
|
||||
.await
|
||||
.expect("inspection should progress after target lock release")
|
||||
.expect("heal task should not panic")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
|
||||
|
||||
@@ -5155,11 +5155,6 @@ mod tests {
|
||||
let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
|
||||
let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
|
||||
if !sharded {
|
||||
crate::bucket::quota::reservation::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap quota with one disk offline");
|
||||
}
|
||||
|
||||
let payload = vec![0x51; 4096];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
@@ -21012,299 +21007,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_retained_null_version(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
protection: &str,
|
||||
latest_is_marker: bool,
|
||||
) {
|
||||
let metadata = if protection == "legal-hold" {
|
||||
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
|
||||
} else {
|
||||
HashMap::from([
|
||||
("x-amz-object-lock-mode".to_string(), protection.to_string()),
|
||||
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
|
||||
])
|
||||
};
|
||||
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut original,
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed retained null version");
|
||||
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed newer unprotected UUID version");
|
||||
if latest_is_marker {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a UUID delete marker above the retained null version");
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained null version must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read retained null version");
|
||||
assert_eq!(body, b"protected null version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_delete_checks_actual_target() {
|
||||
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let temp = tempfile::tempdir().expect("create null-version delete store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let configs = [
|
||||
VersioningConfiguration::default(),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
for (case, config) in configs.into_iter().enumerate() {
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
|
||||
"batch delete must judge the retained null version: {object}: {errors:?}"
|
||||
);
|
||||
let error = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("suspended single delete must also judge the retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
if protection == "GOVERNANCE" {
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
|
||||
bypass_governance: true,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
|
||||
let removed = store
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match removed {
|
||||
Ok(marker) => {
|
||||
assert!(marker.delete_marker, "only a null delete marker may replace the payload");
|
||||
assert_eq!(marker.size, 0);
|
||||
assert!(marker.data_dir.is_none());
|
||||
}
|
||||
Err(error) => assert!(
|
||||
matches!(
|
||||
error,
|
||||
StorageError::MethodNotAllowed
|
||||
| StorageError::ObjectNotFound(_, _)
|
||||
| StorageError::VersionNotFound(_, _, _)
|
||||
),
|
||||
"unexpected bypass result: {error:?}"
|
||||
),
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.get_object_reader(
|
||||
&bucket,
|
||||
&object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized deletion must make the retained null payload unreadable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
|
||||
let temp = tempfile::tempdir().expect("create null-version overwrite store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
|
||||
let error = store
|
||||
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("PUT must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
|
||||
let upload = store
|
||||
.clone()
|
||||
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
|
||||
let part = store
|
||||
.clone()
|
||||
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart replacement");
|
||||
let error = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
|
||||
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
&object,
|
||||
&mut next_version,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a new UUID version must remain allowed");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -423,6 +423,8 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
!is_meta_bucketname(&oi.bucket)
|
||||
}
|
||||
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
@@ -454,6 +456,7 @@ mod rebalance;
|
||||
pub(crate) use rebalance::PoolSpaceForTest;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub(crate) use peer::admit_local_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
|
||||
@@ -273,28 +273,6 @@ async fn pause_data_movement_multipart_before_selected_completion(bucket: &str)
|
||||
}
|
||||
}
|
||||
|
||||
async fn pool_has_multipart_upload_for_key(
|
||||
pool: &crate::core::sets::Sets,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = stream::iter(pool.disk_set.iter().cloned())
|
||||
.map(|set| async move {
|
||||
set.has_multipart_upload_for_key(bucket, object, expected_incarnation_id)
|
||||
.await
|
||||
})
|
||||
.buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let mut found = false;
|
||||
for result in results {
|
||||
// Preserve errors from every set instead of hiding them behind a hit.
|
||||
found |= result?;
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
async fn list_pool_multipart_uploads_for_incarnation(
|
||||
pool: &crate::core::sets::Sets,
|
||||
bucket: &str,
|
||||
@@ -512,8 +490,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ListPartsInfo> {
|
||||
check_list_parts_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
@@ -644,8 +620,6 @@ impl ECStore {
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_object_args(bucket, object, opts.data_movement)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -701,7 +675,21 @@ impl ECStore {
|
||||
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
if pool_has_multipart_upload_for_key(pool, bucket, object, opts.expected_bucket_incarnation_id).await? {
|
||||
let res = list_pool_multipart_uploads_for_incarnation(
|
||||
pool,
|
||||
bucket,
|
||||
&MultipartUploadListRequest {
|
||||
prefix: object.to_string(),
|
||||
key_marker: None,
|
||||
upload_id_marker: None,
|
||||
delimiter: None,
|
||||
max_uploads: MAX_UPLOADS_LIST,
|
||||
expected_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !res.uploads.is_empty() {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
@@ -784,8 +772,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
@@ -820,8 +806,6 @@ impl ECStore {
|
||||
.part_number
|
||||
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
@@ -854,8 +838,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<MultipartInfo> {
|
||||
check_list_parts_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
if self.single_pool() {
|
||||
@@ -889,8 +871,6 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
@@ -1072,8 +1052,6 @@ impl ECStore {
|
||||
for upload_id in upload_ids {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
}
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart abort requires data_movement options"));
|
||||
}
|
||||
@@ -1194,7 +1172,7 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_write_precondition(
|
||||
pub(crate) async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1232,34 +1210,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
// The set-level commit judges the null version this completion
|
||||
// replaces; give it the same authoritative snapshot a PUT carries.
|
||||
let expected_incarnation_id = opts
|
||||
.expected_bucket_incarnation_id
|
||||
.ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
|
||||
let lifecycle_fence = opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
|
||||
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
|
||||
Some(snapshot) => Arc::clone(snapshot),
|
||||
None => {
|
||||
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
|
||||
return Err(Error::other(
|
||||
"Object Lock snapshot does not hold valid target bucket generation and configuration fences",
|
||||
));
|
||||
}
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
opts.object_lock_config_snapshot = Some(snapshot);
|
||||
}
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -1368,8 +1319,6 @@ impl ECStore {
|
||||
let request_opts = opts;
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
@@ -1491,362 +1440,6 @@ mod tests {
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_multipart_pool_selection_uses_only_an_exact_key_upload() {
|
||||
use crate::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
dirs.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("exact-multipart-pool-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("initialize local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("initialize two pools");
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "exact-pool-selection";
|
||||
let other_bucket = "other-pool-selection";
|
||||
for bucket in [bucket, other_bucket] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
for (upload_bucket, key) in [(bucket, "report-long"), (bucket, "report/"), (other_bucket, "report")] {
|
||||
let (opts, _guard) = store
|
||||
.guard_multipart_bucket_incarnation(upload_bucket, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pin upload incarnation");
|
||||
store.pools[0]
|
||||
.new_multipart_upload(upload_bucket, &encode_dir_object(key), &opts)
|
||||
.await
|
||||
.expect("seed nonmatching upload in first pool");
|
||||
}
|
||||
let (opts, _guard) = store
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pin target incarnation");
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", opts.expected_bucket_incarnation_id)
|
||||
.await
|
||||
.expect("first pool contains only other keys")
|
||||
);
|
||||
store.pools[1]
|
||||
.new_multipart_upload(bucket, "report", &opts)
|
||||
.await
|
||||
.expect("seed exact upload in second pool");
|
||||
let (created, pool_index, incarnation) = store
|
||||
.handle_new_multipart_upload_with_pool_idx(bucket, "report", &ObjectOptions::default(), None)
|
||||
.await
|
||||
.expect("create another upload for the exact key");
|
||||
assert_eq!(pool_index, 1, "prefix matches in the first pool must not change placement");
|
||||
assert_eq!(incarnation, opts.expected_bucket_incarnation_id);
|
||||
assert!(!created.upload_id.is_empty());
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", incarnation)
|
||||
.await
|
||||
.expect("new upload must not appear in first pool")
|
||||
);
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[1], bucket, "report", Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("old bucket incarnations must not affect placement")
|
||||
);
|
||||
cancel.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_directory_keys_preserve_the_bare_object_and_round_trip() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use sha2::Digest;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
for versioned in [false, true] {
|
||||
let bucket = if versioned {
|
||||
"directory-mpu-versioned"
|
||||
} else {
|
||||
"directory-mpu-unversioned"
|
||||
};
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
if versioned {
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
"versioning.xml",
|
||||
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
versioned,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
let original = store
|
||||
.put_object(
|
||||
bucket,
|
||||
"protected",
|
||||
&mut PutObjReader::from_vec(b"private original".to_vec()),
|
||||
&ObjectOptions {
|
||||
user_defined: HashMap::from([(
|
||||
rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(),
|
||||
"classification=secret".to_string(),
|
||||
)]),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write protected bare object");
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, "protected/", &opts)
|
||||
.await
|
||||
.expect("create directory upload");
|
||||
let part = store
|
||||
.put_object_part(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::from_vec(b"directory payload".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("write directory part");
|
||||
store
|
||||
.put_object_part(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
2,
|
||||
&mut PutObjReader::from_vec(b"unselected private bytes".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("write an unselected part");
|
||||
let upload_sha = hex_simd::encode_to_string(
|
||||
sha2::Sha256::digest(format!("{bucket}/protected__XLDIR__").as_bytes()),
|
||||
hex_simd::AsciiCase::Lower,
|
||||
);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
for dir in &dirs {
|
||||
let staging = dir
|
||||
.path()
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid);
|
||||
let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(staging.join("xl.meta")).expect("upload metadata"))
|
||||
.expect("decode upload metadata");
|
||||
let versions = metadata
|
||||
.get_file_info_versions(bucket, "protected__XLDIR__", true)
|
||||
.expect("upload file info");
|
||||
let data_dir = versions
|
||||
.versions
|
||||
.first()
|
||||
.expect("upload version")
|
||||
.data_dir
|
||||
.expect("staging data directory");
|
||||
assert!(
|
||||
staging.join(data_dir.to_string()).join("part.2").is_file(),
|
||||
"unselected part exists before completion"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.get_multipart_info(bucket, "protected/", &upload.upload_id, &opts)
|
||||
.await
|
||||
.expect("upload info")
|
||||
.object,
|
||||
"protected/"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.list_object_parts(bucket, "protected/", &upload.upload_id, None, 10, &opts)
|
||||
.await
|
||||
.expect("list parts")
|
||||
.object,
|
||||
"protected/"
|
||||
);
|
||||
let uploads = store
|
||||
.list_multipart_uploads(bucket, "protected/", None, None, None, 10)
|
||||
.await
|
||||
.expect("list logical prefix");
|
||||
assert_eq!(uploads.uploads.len(), 1);
|
||||
assert_eq!(uploads.uploads[0].object, "protected/");
|
||||
assert_eq!(crate::runtime::sources::upload_uuid_suffix(&uploads.uploads[0].upload_id), upload_uuid,);
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete directory upload");
|
||||
assert_eq!(completed.name, "protected/");
|
||||
for (key, expected) in [
|
||||
("protected", b"private original".as_slice()),
|
||||
("protected/", b"directory payload".as_slice()),
|
||||
] {
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read distinct object");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read body");
|
||||
assert_eq!(bytes, expected);
|
||||
}
|
||||
let preserved = store
|
||||
.get_object_info(bucket, "protected", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("bare metadata");
|
||||
assert_eq!(preserved.version_id, original.version_id);
|
||||
assert_eq!(preserved.etag, original.etag);
|
||||
assert_eq!(preserved.user_tags.as_str(), "classification=secret");
|
||||
for dir in &dirs {
|
||||
assert!(dir.path().join(bucket).join("protected/xl.meta").is_file());
|
||||
assert!(dir.path().join(bucket).join("protected__XLDIR__/xl.meta").is_file());
|
||||
let data = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("protected__XLDIR__")
|
||||
.join(completed.data_dir.expect("multipart data directory").to_string());
|
||||
assert!(data.join("part.1").is_file());
|
||||
assert!(!data.join("part.2").exists(), "unselected part must never be published");
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
.exists(),
|
||||
"original staging must be cleaned"
|
||||
);
|
||||
}
|
||||
let abandoned = store
|
||||
.new_multipart_upload(bucket, "protected/", &opts)
|
||||
.await
|
||||
.expect("create abort control");
|
||||
store
|
||||
.abort_multipart_upload(bucket, "protected/", &abandoned.upload_id, &opts)
|
||||
.await
|
||||
.expect("abort encoded upload");
|
||||
assert!(
|
||||
store
|
||||
.get_multipart_info(bucket, "protected/", &abandoned.upload_id, &opts)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, "protected/", &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn new_control_character_keys_are_rejected_without_trapping_legacy_objects() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "xml-key-admission";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
|
||||
let key = format!("bad-{}", char::from(byte));
|
||||
assert!(
|
||||
store
|
||||
.put_object(bucket, &key, &mut PutObjReader::from_vec(b"body".to_vec()), &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.new_multipart_upload(bucket, &key, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(bucket).join(&key).exists());
|
||||
}
|
||||
}
|
||||
let legacy_key = "legacy-\u{1}";
|
||||
store.pools[0]
|
||||
.get_disks_by_key(legacy_key)
|
||||
.put_object(
|
||||
bucket,
|
||||
legacy_key,
|
||||
&mut PutObjReader::from_vec(b"legacy payload".to_vec()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed a pre-upgrade object below new-key admission");
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, legacy_key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy GET stays usable");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("legacy bytes");
|
||||
assert_eq!(bytes, b"legacy payload");
|
||||
drop(reader);
|
||||
store
|
||||
.delete_object(bucket, legacy_key, ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy DELETE stays usable");
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, legacy_key, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
fn mp(object: &str, upload_id: &str) -> MultipartInfo {
|
||||
MultipartInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
+1574
-621
File diff suppressed because it is too large
Load Diff
@@ -85,6 +85,230 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Prepare an inert user-object reservation on the currently admitted disk.
|
||||
/// An unclaimed token expires and does not hold the object's metadata lock.
|
||||
#[cfg(test)]
|
||||
pub async fn reserve_local_object_metadata(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare before the caller acquires bucket lifecycle or object locks.
|
||||
/// The token holds the bucket publication fence through physical completion.
|
||||
pub async fn reserve_local_object_metadata_at_incarnation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_at_incarnation_with_ttl(disk_ref, bucket, object, incarnation, timeout, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare a short-lived token for peers that may abandon the response.
|
||||
pub async fn reserve_local_object_metadata_short(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_at_incarnation_with_ttl(
|
||||
disk_ref,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
timeout,
|
||||
Some(std::time::Duration::from_secs(2)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_at_incarnation_with_ttl(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
ttl: Option<std::time::Duration>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if self.ctx.lock_manager().is_disabled()
|
||||
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|
||||
|| is_meta_bucketname(bucket)
|
||||
|| incarnation.is_nil()
|
||||
|| timeout.is_zero()
|
||||
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
|
||||
let mut cached = slot.owner.lock().await;
|
||||
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
|
||||
Some(owner) => owner,
|
||||
None => {
|
||||
// Bucket deletion takes publication before lifecycle and metadata.
|
||||
let lock = self
|
||||
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let publication = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
// Older peers may acquire lifecycle without publication.
|
||||
let lifecycle = self
|
||||
.acquire_bucket_incarnation_fence(bucket, incarnation)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
|
||||
if !owner.is_reusable() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
*cached = Arc::downgrade(&owner);
|
||||
owner
|
||||
}
|
||||
};
|
||||
drop(cached);
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner), ttl)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_with_owner(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
|
||||
ttl: Option<std::time::Duration>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
match ttl {
|
||||
Some(ttl) => {
|
||||
wrapper
|
||||
.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
|
||||
.await
|
||||
}
|
||||
None => wrapper.prepare_object_metadata_with_owner(bucket, object, owner).await,
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
/// Refresh an inert token on its original disk and object.
|
||||
pub async fn renew_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.renew_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token on its original disk and object.
|
||||
pub async fn activate_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed token on the currently admitted disk.
|
||||
/// A token already claimed by a physical mutation cannot be cancelled.
|
||||
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
Ok(wrapper.cancel_object_metadata_reservation(token))
|
||||
}
|
||||
|
||||
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
|
||||
/// The token is bound to the admitted disk instance and destination. A claimed
|
||||
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
|
||||
/// The caller must retain the bucket lifecycle fence until the physical tail
|
||||
/// finishes; the reservation does not identify a bucket incarnation.
|
||||
pub async fn rename_local_data_reserved(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
token: Uuid,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if token.is_nil()
|
||||
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|
||||
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|
||||
|| is_meta_bucketname(destination.0)
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
@@ -270,6 +494,7 @@ impl ECStore {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -351,7 +576,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
pub(crate) async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
@@ -783,6 +1008,116 @@ mod tests {
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let dir = tempfile::tempdir().expect("disk root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, dir.path(), id).await;
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("object", version, b"reserved");
|
||||
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
|
||||
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
|
||||
.await
|
||||
.expect("user bucket");
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(
|
||||
&disk_ref,
|
||||
"target-bucket/../.rustfs.sys",
|
||||
"object",
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
let token = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve current disk");
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
store
|
||||
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
|
||||
.await
|
||||
.expect("activate token");
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
token,
|
||||
)
|
||||
.await
|
||||
.expect("reserved publication");
|
||||
assert!(
|
||||
!store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("consumed token")
|
||||
);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read reserved publication");
|
||||
assert_eq!(read.data, fi.data);
|
||||
|
||||
let stale = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve before disk replacement");
|
||||
let replacement = target_disk(&ctx, dir.path(), id).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
stale,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
));
|
||||
let after = replacement
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement retains publication");
|
||||
assert_eq!(after.data, fi.data);
|
||||
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
|
||||
@@ -604,6 +604,24 @@ impl ECStore {
|
||||
Ok(idx)
|
||||
}
|
||||
|
||||
/// A speculative hint only: callers must repeat the locked selection before committing.
|
||||
pub(super) async fn get_pool_idx_for_reservation_hint(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: true,
|
||||
skip_decommissioned: true,
|
||||
skip_rebalancing: true,
|
||||
..Default::default()
|
||||
};
|
||||
match self.get_pool_idx_existing_with_opts(bucket, object, &opts).await {
|
||||
Ok(idx) => Ok(idx),
|
||||
Err(err) if is_err_object_not_found(&err) => {
|
||||
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
|
||||
let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await {
|
||||
Ok(res) => res,
|
||||
|
||||
@@ -20,230 +20,14 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::contract_compat::ECStore;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, DeleteBucketOptions, Error, HashReader, MakeBucketOptions, MultipartOperations,
|
||||
NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause,
|
||||
init_bucket_metadata_sys, isolated_store_over_temp_disks,
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct ObservedPutBody {
|
||||
inner: tokio::io::DuplexStream,
|
||||
polled: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncRead for ObservedPutBody {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
self.polled.notify_one();
|
||||
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn destructive_put_takes_bucket_locks_only_after_its_body_is_staged() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
for (object, version_id) in [("object", None), ("directory/", Some(Uuid::nil().to_string()))] {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-locks";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let encoded_object = if object.ends_with('/') { "directory__XLDIR__" } else { object };
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, encoded_object, PutObjectCommitPause::BeforeQuotaRename);
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
version_id,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified())
|
||||
.await
|
||||
.expect("PUT must be waiting for client data");
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("metadata lock");
|
||||
drop(
|
||||
timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5)))
|
||||
.await
|
||||
.expect("body must not block metadata writers")
|
||||
.expect("metadata write guard"),
|
||||
);
|
||||
drop(
|
||||
timeout(Duration::from_secs(2), store.lock_bucket_for_recursive_delete(bucket))
|
||||
.await
|
||||
.expect("body must not retain lifecycle guard")
|
||||
.expect("lifecycle writer"),
|
||||
);
|
||||
sender.write_all(b"body").await.expect("finish body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
timeout(Duration::from_secs(5), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("staged PUT reaches commit");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending(), "commit must fence metadata updates");
|
||||
let lifecycle = store.lock_bucket_for_recursive_delete(bucket);
|
||||
tokio::pin!(lifecycle);
|
||||
assert!(futures::poll!(&mut lifecycle).is_pending(), "commit must fence bucket deletion");
|
||||
barrier.release();
|
||||
let release_metadata = async {
|
||||
drop(writer.await.expect("metadata writer"));
|
||||
};
|
||||
let release_lifecycle = async {
|
||||
drop(lifecycle.await.expect("lifecycle writer"));
|
||||
};
|
||||
timeout(Duration::from_secs(5), async {
|
||||
let (put, (), ()) = tokio::join!(put, release_metadata, release_lifecycle);
|
||||
put.expect("owner task").expect("commit");
|
||||
})
|
||||
.await
|
||||
.expect("PUT and queued writers finish");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_put_cannot_publish_into_a_recreated_bucket() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-generation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
|
||||
store
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove old bucket while PUT stalls");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
sender.write_all(b"body").await.expect("finish body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
let error = timeout(Duration::from_secs(5), put)
|
||||
.await
|
||||
.expect("commit gate")
|
||||
.expect("owner")
|
||||
.expect_err("old request generation must not publish");
|
||||
assert!(
|
||||
error.to_string().contains("generation") || matches!(error, Error::BucketNotFound(..)),
|
||||
"{error:?}"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_put_rechecks_current_object_lock_at_commit() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-retention";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("locked bucket");
|
||||
let mut old = PutObjReader::from_vec(b"old".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut old, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("unlocked predecessor");
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
|
||||
let mut protected = PutObjReader::from_vec(b"protected".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut protected,
|
||||
&ObjectOptions {
|
||||
user_defined: std::collections::HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("install a protected predecessor while the client stalls");
|
||||
sender.write_all(b"body").await.expect("finish stale body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
let error = timeout(Duration::from_secs(5), put)
|
||||
.await
|
||||
.expect("commit gate")
|
||||
.expect("task")
|
||||
.expect_err("pre-body state cannot admit an overwrite of the newly protected version");
|
||||
assert!(matches!(error, Error::PrefixAccessDenied(..)), "{error:?}");
|
||||
let info = store
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("protected object remains");
|
||||
assert_eq!(info.size, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(false).await;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user