Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue c8a646b67c fix(ecstore): order foreground writes during disk recovery 2026-11-02 05:45:21 +08:00
342 changed files with 15305 additions and 36891 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+1 -1
View File
@@ -1,2 +1,2 @@
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
sha256-linux=01e1f5569be60674e30d9e30a9961c5c53a6cea6ca1f7913218f2adf7291d574
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
+1 -1
View File
@@ -1 +1 @@
sha256=f794c004e5952147c79dadbe6b7c84804bc986f34a8142731fac8864004e5130
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
+11 -5
View File
@@ -417,6 +417,12 @@ threads-required = "num-test-threads"
filter = 'package(rustfs-heal) & (binary(mrf_partial_write_test) | (binary(mrf_pipeline_test) & test(/^journal_replay_survives_/)) | test(=admin_selector_survives_sigkill_with_original_scope_and_receipts))'
threads-required = "num-test-threads"
# Bound the whole recovery fixture even if a synchronous wait prevents its
# per-case async deadline from firing.
[[profile.ci.overrides]]
filter = 'package(rustfs-heal) & binary(heal_integration_test) & test(=serial_tests::test_admin_bucket_directory_recovery_from_task_entry)'
slow-timeout = { period = "60s", terminate-after = 5, grace-period = "10s" }
# Native scanner backlog fixtures include 24-set and pool-expansion cases.
# Reserve capacity across test processes while preserving each scenario's fixed
# budget, assertions, and internal concurrency.
@@ -701,7 +707,7 @@ test-group = 'ecstore-serial-flaky'
[profile.e2e-smoke]
default-filter = """
package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/)
| test(/^conditional_delete_objects_test::/)
@@ -827,7 +833,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Nightly and manual-dispatch lane owned by
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -970,10 +976,10 @@ threads-required = "num-test-threads"
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
test-group = 'e2e-vault'
# Keep the storage-heavy smoke cases isolated in the full suite too. They
# start multiple server fixtures and issue many sequential S3 operations.
# Keep storage-heavy cases isolated in the full suite too. The large pagination
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
threads-required = "num-test-threads"
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
+3 -3
View File
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
| File | Description | Usage |
| :--- | :--- | :--- |
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
### Docker Compose Configurations
-2
View File
@@ -32,8 +32,6 @@ updates:
- "overtrue"
- "majinghe"
ignore:
- dependency-name: "pulsar"
versions: [ "6.9.x" ]
- dependency-name: "object_store"
versions: [ "0.13.x" ]
- dependency-name: "libunftp"
+5 -2
View File
@@ -769,10 +769,12 @@
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"workflow_dispatch",
"schedule"
],
@@ -847,11 +849,12 @@
"workflow": ".github/workflows/e2e-upgrade.yml",
"name": "Upgrade Compatibility",
"roles": [
"pr",
"periodic",
"manual",
"release"
"manual"
],
"events": [
"pull_request",
"push",
"schedule",
"workflow_dispatch"
+1 -60
View File
@@ -113,7 +113,7 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
@@ -497,63 +497,6 @@ jobs:
- name: Check offline enrollment E2E root boundary
run: ./scripts/check_offline_enrollment_e2e.sh
fips-provider:
name: FIPS provider boundaries
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
mode: [default, fips]
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
CARGO_BUILD_JOBS: "2"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-fips-provider
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install FIPS build tools
if: matrix.mode == 'fips'
run: sudo apt-get install -y cmake golang clang libclang-dev
- name: Verify provider installation
shell: bash
run: |
args=()
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
args+=(--features fips)
fi
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
fi
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
fi
for test_name in "${tests[@]}"; do
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
done
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
# Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1454,7 +1397,6 @@ jobs:
- workspace-shard-evidence
- core-contract
- connect-feature-boundaries
- fips-provider
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
@@ -1482,7 +1424,6 @@ jobs:
needs:
- classify-changes
- connect-feature-boundaries
- fips-provider
- required-checks
- typos
- quick-checks
+28 -2
View File
@@ -17,8 +17,8 @@
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
# and manual dispatches execute the same fail-closed suite.
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".config/e2e-distributed-selection.txt"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
+18 -7
View File
@@ -12,19 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Single-node upgrade and rollback lane.
#
# The weekly Monday schedule, version tags, and workflow_dispatch run this
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
name: Upgrade Compatibility
on:
pull_request:
paths:
- ".github/workflows/e2e-upgrade.yml"
- "crates/e2e_test/src/common.rs"
- "crates/e2e_test/src/distributed/**"
- "crates/e2e_test/src/fake_s3_target/**"
- "crates/e2e_test/src/lib.rs"
- "crates/e2e_test/src/replication_extension_test.rs"
- "crates/e2e_test/src/rpc_fault_proxy.rs"
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/kms/**"
- "crates/storage-api/**"
- "rustfs/**"
- "Cargo.lock"
push:
tags:
- "[0-9]*.[0-9]*.[0-9]*"
schedule:
# 03:17 UTC every Monday.
- cron: "17 3 * * 1"
workflow_dispatch:
@@ -33,7 +44,7 @@ permissions:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
+7 -279
View File
@@ -87,206 +87,18 @@ jobs:
install-build-packaging-tools: 'false'
install-test-tools: 'false'
- name: Download static console assets
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
# Packaging tools are off on this lane, so setup does not install unzip.
if ! command -v unzip >/dev/null 2>&1; then
sudo apt-get update -qq
sudo apt-get install -y -qq unzip
fi
if ! command -v unzip >/dev/null 2>&1; then
echo "unzip is required to extract console assets" >&2
exit 1
fi
mkdir -p ./rustfs/static
verify_sha256() {
local expected="$1"
local file="$2"
local actual=""
local actual_line=""
if command -v sha256sum >/dev/null 2>&1; then
actual_line=$(sha256sum -- "$file") || return 1
elif command -v shasum >/dev/null 2>&1; then
actual_line=$(shasum -a 256 -- "$file") || return 1
else
echo "No SHA-256 verification tool found" >&2
return 1
fi
actual="${actual_line%%[[:space:]]*}"
if [ "$actual" = "$expected" ]; then
echo "SHA256 verified OK: $actual"
return 0
else
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
return 1
fi
}
download_console_assets() {
local curl_bin="curl"
local python_bin="python3"
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
local console_json="console-release.json"
local console_url
local console_sha256
local curl_auth_args=()
console_tag=""
chmod +w ./rustfs/static/LICENSE || true
if ! command -v "$python_bin" >/dev/null 2>&1; then
python_bin="python"
fi
if ! command -v "$python_bin" >/dev/null 2>&1; then
echo "No Python interpreter found for release metadata parsing" >&2
return 1
fi
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
fi
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
import json
import re
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
release = json.load(handle)
tag = release.get("tag_name", "") or "unknown"
for asset in release.get("assets", []):
name = asset.get("name", "")
digest = asset.get("digest", "")
url = asset.get("browser_download_url", "")
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
sha256 = digest.split(":", 1)[1]
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
break
else:
raise SystemExit("no console zip asset with sha256 digest found")
PY
) || return 1
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
echo "Console release metadata is missing URL or SHA-256 digest" >&2
return 1
fi
echo "Console release: ${console_tag}"
echo "Downloading console asset: ${console_url}"
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
verify_sha256 "$console_sha256" console.zip || return 2
unzip -o console.zip -d ./rustfs/static || return 2
}
if download_console_assets; then
rm -f console.zip console-release.json
else
status=$?
rm -f console.zip console-release.json
if [[ "$status" -eq 2 ]]; then
echo "Console asset integrity verification failed" >&2
fi
echo "Failed to download verified console assets" >&2
exit 1
fi
if [[ ! -s ./rustfs/static/index.html ]]; then
echo "Console asset archive is missing static/index.html" >&2
exit 1
fi
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
- name: Build RustFS
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
shell: bash
run: |
# Force rebuild by touching build.rs so a restored cache cannot
# reuse a rustfs artifact compiled before these console assets existed.
touch rustfs/build.rs
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
- name: Verify glibc symbol baseline
run: |
python3 scripts/release/check_glibc.py 2.31 \
target/x86_64-unknown-linux-gnu/release/rustfs
target/x86_64-unknown-linux-gnu/release/rustfs \
target/x86_64-unknown-linux-gnu/release/rustfs-cli
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
- name: Verify console
shell: bash
run: |
set -euo pipefail
find_free_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
data_dir="$(mktemp -d)"
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
server_pid=""
trap '
if [[ -n "${server_pid:-}" ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
fi
rm -rf "$data_dir"
' EXIT
if [[ ! -x "$binary" ]]; then
echo "rustfs binary not found: ${binary}" >&2
exit 1
fi
api_port="$(find_free_port)"
console_port="$(find_free_port)"
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
"$binary" server \
--address "127.0.0.1:${api_port}" \
--console-enable \
--console-address "127.0.0.1:${console_port}" \
--access-key console-smoke \
--secret-key console-smoke-secret \
"$data_dir" >"$log_file" 2>&1 &
server_pid=$!
response=""
for _ in {1..40}; do
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
if [[ "$response" == 200\ text/html* ]]; then
exit 0
fi
sleep 0.25
done
echo "Console endpoint did not return 200 text/html: $response" >&2
cat "$log_file"
exit 1
- name: Build DEB package
id: deb
shell: bash
@@ -534,12 +346,12 @@ jobs:
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
# engineers can download and install the nightly directly. The branch is
# a single-commit orphan rewritten on every build, which keeps the repo
# small while the latest files stay reachable at stable raw URLs.
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
# The dispatch below runs only after this upload and the immutable R2
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
# which still starts from this workflow's workflow_run completion.
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
- name: Publish packages to auto-testing release assets
env:
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -548,90 +360,6 @@ jobs:
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
run: bash scripts/release/publish_nightly_assets.sh
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
# or fine-grained Contents write (the permission GitHub checks for
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
# step, and therefore this job, so the scheduled failure alert fires.
# GitHub allows at most 10 top-level client_payload properties, so the
# schema 2 candidate object is nested under `candidate` and the release
# fields sit beside it.
- name: Dispatch nightly candidate to auto-testing
if: ${{ steps.publish.outputs.candidate_file != '' }}
env:
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
SOURCE_REPO: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
: "${DEB_DATE:?DEB_DATE is required}"
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
--arg event_type "nightly-candidate-ready" \
--arg source_repo "${SOURCE_REPO}" \
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
--arg channel "nightly" \
--arg deb_date "${DEB_DATE}" \
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
'{
event_type: $event_type,
source_repo: $source_repo,
latest_alias_url: $latest_alias_url,
auto_testing_release: $auto_testing_release,
channel: $channel,
deb_date: $deb_date,
nightly_workflow: $nightly_workflow,
candidate: $candidate[0]
}' > "${payload}"
if ! jq -e '
.event_type == "nightly-candidate-ready"
and .source_repo == "rustfs/rustfs"
and .channel == "nightly"
and .deb_date == env.DEB_DATE
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
and .candidate.schema == 2
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
' "${payload}" >/dev/null; then
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
exit 1
fi
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile client_payload "${payload}" \
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-o "${response}" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "Content-Type: application/json" \
--data-binary @"${body}" \
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
exit 1
}
if [[ "${status}" != "204" ]]; then
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
if [[ -s "${response}" ]]; then
cat "${response}" >&2
fi
exit 1
fi
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
+5 -41
View File
@@ -92,17 +92,6 @@ jobs:
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
run: |
set -euo pipefail
# Scheduled runs carry no inputs and the resolver only derives a
# build from a workflow_run payload (which a schedule cannot
# provide), so resolve the newest successful nightly build
# explicitly before invoking it.
if [ -z "${BUILD_RUN_ID:-}" ]; then
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
fi
python3 scripts/resolve_functional_candidate.py
python3 - <<'PY'
import json
@@ -110,15 +99,15 @@ jobs:
from pathlib import Path
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
root.mkdir(parents=True, exist_ok=True)
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = chain["candidate"]["manifest"]
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = candidate["manifest"]
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("testing_sha=" + chain["testing_sha"] + "\n")
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
output.write("package_url=" + manifest["package_url"] + "\n")
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
@@ -132,26 +121,6 @@ jobs:
path: auto-testing
persist-credentials: false
- name: Prepare validation Python runtime
# The pool / ilm / replication validation runners require a managed
# Python, exactly like the chain's validation job provides.
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install validation Python requirements
# Case scripts call python3 and import boto3 at startup. Install the
# functional chain's client into a per-run env so that interpreter
# sees it without changing the runner's shared Python.
run: |
set -euo pipefail
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
test ! -e "${validation_python}"
python3 -m venv "${validation_python}"
"${validation_python}/bin/python" -m pip install \
-r auto-testing/scripts/validation_ilm/requirements.txt
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
- name: Select sweep cases
id: select
env:
@@ -255,8 +224,7 @@ jobs:
# topology, this purge guarantees the next case starts clean even
# when the previous one died mid-run.
for node in ${RUSTFS_NODES}; do
# -n keeps ssh from consuming the case list this loop reads from stdin
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
@@ -275,7 +243,6 @@ jobs:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
run: |
set -euo pipefail
python3 - <<'PY'
@@ -384,10 +351,7 @@ jobs:
${{ runner.temp }}/validation-sweep/sweep-cases.txt
${{ runner.temp }}/validation-sweep/candidate.json
${{ runner.temp }}/validation-sweep/results/
${{ runner.temp }}/validation-sweep/run/
# warn (not error): an early infra failure legitimately leaves no
# per-case evidence, and the failing step is already red.
if-no-files-found: warn
if-no-files-found: error
retention-days: 30
- name: Cleanup environment (after)
-1
View File
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
### Security
Generated
+195 -398
View File
File diff suppressed because it is too large Load Diff
+17 -15
View File
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-compression = { version = "0.4.51" }
async-trait = "0.1.92"
async-nats = { version = "0.51.0", default-features = false }
axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
memchr = "2.8.3"
minlz = "1.2.3"
reqwest = "0.13.5"
rustfs-kafka-async = { version = "1.4.1" }
rustfs-kafka-async = { version = "1.4.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.54.1" }
tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -191,7 +191,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
# Serialization and Data Formats
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
bytes = { version = "1.12.1" }
csv-core = "0.1.13"
bytesize = "2.7.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
@@ -213,13 +212,12 @@ serde_urlencoded = "0.7.1"
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
aws-lc-rs = { version = "1.19.0", default-features = false }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.1.0" }
openidconnect = { default-features = false, version = "4.1.0" }
openidconnect = { default-features = false, version = "4.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
@@ -236,7 +234,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "0.2.39" }
jiff = { version = "0.2.38" }
time = { version = "0.3.55" }
# Database
@@ -255,9 +253,9 @@ atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.0" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.129.0" }
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
aws-sdk-sts = { default-features = false, version = "1.124.0" }
aws-sdk-kms = { default-features = false, version = "1.128.0" }
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
aws-sdk-sts = { default-features = false, version = "1.123.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.19.2" }
@@ -339,7 +337,7 @@ strum = { version = "0.28.0" }
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
test-case = "3.4.0"
thiserror = "2.0.21"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
@@ -358,13 +356,13 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.3.0"
zip = "9.1.3"
zstd = "0.14.3"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.7", default-features = false }
dial9 = { version = "0.5.6", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -379,16 +377,16 @@ libunftp = { version = "0.24.0" }
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.1" }
russh = { version = "0.65.0" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.3" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.3", default-features = false }
hotpath = { version = "0.30.2", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
@@ -429,3 +427,7 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
zstd = { path = "crates/zstd-compat" }
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
+3 -11
View File
@@ -16,7 +16,6 @@ FROM alpine:3.24.2 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apk add --no-cache ca-certificates curl jq unzip
WORKDIR /build
@@ -70,9 +69,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
FROM alpine:3.24.2
@@ -99,18 +95,14 @@ RUN apk upgrade --no-cache && \
ca-certificates \
coreutils \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN addgroup -g 10001 -S rustfs && \
adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -132,6 +124,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+4 -13
View File
@@ -12,10 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM rust:1.99.1-trixie
ARG TARGETARCH
ARG RC_RELEASE=latest
FROM rust:1.97.1-trixie
RUN set -eux; \
export DEBIAN_FRONTEND=noninteractive; \
@@ -24,7 +21,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -40,7 +36,6 @@ COPY . .
RUN ./scripts/static.sh
RUN cargo run --bin gproto
RUN cargo build --release --locked --bin rustfs
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
RUN set -eux; \
groupadd -g 10001 rustfs; \
@@ -55,12 +50,8 @@ RUN set -eux; \
chown -R rustfs:rustfs /data /logs /app; \
chmod 0750 /data /logs
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
@@ -72,5 +63,5 @@ ENV RUSTFS_VOLUMES="/data" \
EXPOSE 9000 9001
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
+3 -11
View File
@@ -16,7 +16,6 @@ FROM ubuntu:26.04 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
@@ -77,9 +76,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
FROM ubuntu:26.04
ARG RELEASE=latest
@@ -103,18 +99,14 @@ RUN apt-get update && apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN groupadd -g 10001 rustfs && \
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -139,6 +131,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+7 -19
View File
@@ -28,19 +28,17 @@ ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES=""
ARG RC_RELEASE=latest
# -----------------------------
# Build stage
# -----------------------------
FROM rust:1.99.1-trixie AS builder
FROM rust:1.97.1-trixie AS builder
# Re-declare args after FROM
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES
ARG RC_RELEASE
# Debug: print platforms
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -54,7 +52,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -169,8 +166,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
;; \
esac
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
# -----------------------------
# Development stage (keeps toolchain)
# -----------------------------
@@ -204,13 +199,10 @@ ENV RUSTFS_VOLUMES="/data" \
# We rely on cargo run to build and run
EXPOSE 9000 9001
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["cargo", "run", "--bin", "rustfs", "--"]
# -----------------------------
@@ -235,7 +227,6 @@ RUN set -eux; \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
coreutils; \
rm -rf /var/lib/apt/lists/*
@@ -257,11 +248,8 @@ RUN set -eux; \
# Copy the freshly built binary and the entrypoint
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
# Default environment (override in docker run/compose as needed)
ENV RUSTFS_VOLUMES="/data" \
@@ -275,5 +263,5 @@ EXPOSE 9000
VOLUME ["/data"]
# Keep root here; entrypoint will drop privileges using chroot --userspec
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
-21
View File
@@ -1,21 +0,0 @@
# RustFS image bootstrap
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
Optional per-bucket files use the bucket name as the filename without an extension:
| Directory | File format | Applied operation |
| --- | --- | --- |
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
-1
View File
@@ -1 +0,0 @@
[]
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-6
View File
@@ -1,6 +0,0 @@
{
"block_public_acls": true,
"ignore_public_acls": true,
"block_public_policy": true,
"restrict_public_buckets": true
}
-1
View File
@@ -1 +0,0 @@
+3 -3
View File
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
/// Default validation mode is "hop_by_hop".
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
/// Environment variable to enable RFC 7239 "Forwarded" header support.
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
/// RFC 7239 support is enabled by default.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
/// Environment variable for the maximum allowed proxy hops.
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
-6
View File
@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
/// Queue-store compression is enabled by default to reduce disk footprint.
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
+3 -3
View File
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -12,249 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::{
Client,
error::{ProvideErrorMetadata, SdkError},
primitives::ByteStream,
};
use std::any::TypeId;
use std::collections::BTreeSet;
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
use tokio::time::Instant;
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: Debug + ProvideErrorMetadata + 'static,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
// The workload uses unique unversioned keys. Repeat only explicitly
// advertised lock backpressure; every step still needs its real result.
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().is_some_and(|error| {
let Some(response) = error.raw_response() else {
return false;
};
let Some(service_error) = error.as_service_error() else {
return false;
};
// HEAD has no XML error body, so its SDK service error can
// lack a code even when the server supplies retry headers.
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|| (service_error.code().is_none()
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
response.status().as_u16() == 503
&& namespace_code
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
&& response.headers().get("retry-after") == Some("1")
});
if !retry {
return result;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await;
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
eprintln!(
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
started.elapsed()
);
}
outcome.map_err(|_| {
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
})
}
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
let actual = output.body.collect().await?.into_bytes();
if actual.as_ref() != expected {
return Err(format!(
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
expected.len(),
sha256_hex(expected),
actual.len(),
sha256_hex(&actual)
)
.into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_http_client::test_util::infallible_client_fn;
use aws_smithy_types::body::SdkBody;
use std::sync::atomic::{AtomicUsize, Ordering};
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
let requests = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&requests);
let transport = infallible_client_fn(move |_| {
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
let body = if code.is_empty() {
body.to_string()
} else {
format!("<Error><Code>{code}</Code></Error>")
};
let mut response = http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len());
if let Some(cause) = cause {
response = response.header("x-rustfs-error-cause", cause);
}
if let Some(retry_after) = retry_after {
response = response.header("retry-after", retry_after);
}
response.body(SdkBody::from(body)).expect("scripted S3 response")
});
let config = crate::common::build_test_s3_config(
"http://contention.example.invalid",
"synthetic-access",
"synthetic-secret",
None,
"contention-test",
);
let client = Client::from_conf(config.to_builder().http_client(transport).build());
(s3_probe_client(&client), requests)
}
fn contention() -> Reply {
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
}
#[tokio::test(start_paused = true)]
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt < 3 {
contention()
} else {
(200, "", None, None, "")
}
});
let standard = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
.build(),
);
standard
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
.await
.expect_err("three advertised lock responses exhaust the standard SDK policy");
assert_eq!(requests.load(Ordering::SeqCst), 3);
requests.store(0, Ordering::SeqCst);
let started = Instant::now();
after_lock_contention("PUT recovery", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await??;
assert_eq!(requests.load(Ordering::SeqCst), 4);
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt == 0 {
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
} else {
(200, "", None, None, "")
}
});
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
assert_eq!(requests.load(Ordering::SeqCst), 2);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
for reply in [
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(403, "AccessDenied", None, None, ""),
(404, "NoSuchKey", None, None, ""),
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "ServiceUnavailable", None, Some("1"), ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
] {
let (client, requests) = client(move |_| reply);
let result = after_lock_contention("PUT fatal response", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await?;
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn persistent_lock_contention_exhausts_the_request_budget() {
let (client, requests) = client(|_| contention());
let started = Instant::now();
let result = after_lock_contention("PUT unavailable", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await;
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
assert!(requests.load(Ordering::SeqCst) > 1);
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
}
#[tokio::test(start_paused = true)]
async fn acknowledged_wrong_bytes_fail_without_retry() {
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
.await
.expect_err("a successful but incorrect read is a consistency failure");
assert!(error.to_string().contains("bytes mismatch"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
// This consistency probe owns its admission policy; unrelated responses
// must not be hidden by the SDK's general-purpose retries.
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
let context = format!("worker={idx} round={round}");
after_lock_contention(&format!("{context} PUT source"), || {
writer
.put_object()
.bucket(&bucket)
.key(&key)
.body(ByteStream::from(body.clone()))
.send()
})
.await??;
put_object(writer, &bucket, &key, body.clone()).await?;
let head = after_lock_contention(&format!("{context} HEAD source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await??;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_workload_bytes(reader, &bucket, &key, &body).await?;
assert_object_bytes(reader, &bucket, &key, &body).await?;
after_lock_contention(&format!("{context} COPY retained"), || {
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
})
.await??;
assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
after_lock_contention(&format!("{context} DELETE source"), || {
writer.delete_object().bucket(&bucket).key(&key).send()
})
.await??;
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await?
.expect_err("deleted source key must not remain visible");
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
for handle in handles {
inventory.extend(handle.await??);
}
assert_eq!(
inventory.len(),
worker_count * rounds,
"every worker round must retain its acknowledged copy"
);
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
})
.await??;
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_workload_bytes(client, &bucket, key, body)
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
-3
View File
@@ -283,9 +283,6 @@ mod ssec_copy_test;
#[cfg(test)]
mod multipart_storage_class_test;
#[cfg(test)]
mod multipart_upload_id_test;
#[cfg(test)]
mod storage_class_capability_test;
@@ -1,175 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Multipart upload ID regression tests.
//!
//! Every operation that addresses an existing multipart upload must report an
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
//! the ID is malformed or is a well-formed ID of an aborted upload.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const BUCKET: &str = "multipart-upload-id";
const KEY: &str = "target.bin";
const COPY_SOURCE_KEY: &str = "copy-source.bin";
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
// the object layer rejects this ID as malformed instead of looking it up.
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
const PART_NUMBER: i32 = 1;
const PART_BODY: &[u8] = b"multipart upload id regression part";
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
where
E: ProvideErrorMetadata + std::fmt::Debug,
{
let Err(error) = result else {
return Some(format!("{operation}: succeeded"));
};
let status = error.raw_response().map(|response| response.status().as_u16());
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
}
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
env.create_test_bucket(BUCKET).await?;
let client = env.create_s3_client();
client
.put_object()
.bucket(BUCKET)
.key(COPY_SOURCE_KEY)
.body(ByteStream::from_static(PART_BODY))
.send()
.await?;
Ok((env, client))
}
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
let upload_part = client
.upload_part()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.body(ByteStream::from_static(PART_BODY))
.send()
.await;
let upload_part_copy = client
.upload_part_copy()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
.send()
.await;
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
let complete = client
.complete_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
.build(),
)
.send()
.await;
let abort = client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await;
let mismatches: Vec<String> = [
no_such_upload_mismatch("UploadPart", upload_part),
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
no_such_upload_mismatch("ListParts", list_parts),
no_such_upload_mismatch("CompleteMultipartUpload", complete),
no_such_upload_mismatch("AbortMultipartUpload", abort),
]
.into_iter()
.flatten()
.collect();
assert!(
mismatches.is_empty(),
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
mismatches.join("\n")
);
}
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
let status = head
.as_ref()
.err()
.and_then(|error| error.raw_response())
.map(|response| response.status().as_u16());
assert_eq!(
status,
Some(OBJECT_NOT_FOUND_STATUS),
"rejected operations must not publish {KEY}: {head:?}"
);
Ok(())
}
#[tokio::test]
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
assert_no_upload_or_object_published(&client).await
}
#[tokio::test]
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await?;
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
assert_no_upload_or_object_published(&client).await
}
}
@@ -576,6 +576,41 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
async fn reserve_object_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn reserve_object_metadata_short(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn activate_object_metadata_reservation(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn cancel_object_metadata_reservation(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_reserved_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_data_with_pool_meta_cas(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
ServerSideEncryption, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
VersioningConfiguration,
};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
init_logging();
let mut source_env = RustFSTestEnvironment::new().await?;
let source_log = format!("{}/source.log", source_env.temp_dir);
source_env.capture_log_path = Some(source_log.clone());
let mut source_env_vars = replication_fast_env();
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
"target did not preserve the source delete-marker version ID"
);
let wait_for_source_writeback = || async {
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
timeout(Duration::from_secs(30), async {
loop {
let response = signed_request(
http::Method::GET,
&metrics_url,
&source_env.access_key,
&source_env.secret_key,
None,
None,
)
.await?;
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
let queued = metrics["queued"]["curr"]["count"]
.as_f64()
.ok_or("missing replication queue count")?;
if metrics["provider_available"] == true && queued == 0.0 {
return Ok::<(), Box<dyn Error + Send + Sync>>(());
}
sleep(Duration::from_millis(50)).await;
}
})
.await?
};
wait_for_source_writeback().await?;
let purged = source_client
.delete_objects()
source_client
.delete_object()
.bucket(source_bucket)
.delete(
Delete::builder()
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(&purged_version_id)
.build()?,
)
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(delete_marker_version_id)
.build()?,
)
.build()?,
)
.key(object_key)
.version_id(&purged_version_id)
.send()
.await?;
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
let target_state = list_replication_state(&target_client, target_bucket).await?;
assert!(
target_state.iter().all(|entry| entry.version_id != purged_version_id),
"target retained the explicitly purged object version"
);
assert!(
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
"target retained the explicitly purged delete marker"
);
assert!(
target_state
.iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
.await?;
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
// Wait for the source workers to finish their final writeback, which runs
// after the remote purge has already become visible (rustfs#9121).
wait_for_source_writeback().await?;
let logs = fs::read_to_string(source_log).await?;
let writeback_failures: Vec<_> = logs
.lines()
.filter(|line| line.contains("apply_replication_delete_state"))
.collect();
assert!(
writeback_failures.is_empty(),
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
);
Ok(())
}
-9
View File
@@ -94,12 +94,3 @@ pub(crate) mod scanner_peer_bucket {
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
pub(crate) use ecstore_rpc::PeerRestClient;
}
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
#[cfg(test)]
pub(crate) mod bucket_tag_fixture {
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
}
+29 -147
View File
@@ -360,26 +360,12 @@ impl OpaMock {
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, action, bucket, department);
Ok(())
}
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
assert_eq!(payload["input"]["resource"]["object"], object);
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
Ok(())
}
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
let input = &payload["input"];
assert_eq!(input["identity"]["account"], "opabuckettags");
assert_eq!(input["identity"]["is_owner"], false);
assert_eq!(input["action"], action);
assert_eq!(input["resource"]["bucket"], bucket);
let expected = department.map(|value| serde_json::json!([value]));
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
Ok(())
}
async fn wait_for_validation(&mut self) -> TestResult {
@@ -692,61 +678,22 @@ async fn test_list_buckets_opa_contract() -> TestResult {
Ok(())
}
/// Stop and reap the directory's owning server before opening the fixture store.
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
use crate::storage_api::bucket_tag_fixture::{
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
};
let server = env
.process
.as_mut()
.ok_or("missing server process before offline fixture write")?;
server.kill()?;
server.wait()?;
env.process = None;
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
let context = std::sync::Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
let _cancel = context
.background_cancel_token()
.expect("store binds its shutdown token")
.drop_guard();
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
let incarnation = metadata.bucket_incarnation_id;
let policy = metadata.policy_config_json.clone();
assert!(!incarnation.is_nil());
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
metadata.tagging_config_xml = xml.to_vec();
metadata.save_with_store(store.clone()).await?;
let persisted = load_bucket_metadata(store, bucket).await?;
assert_eq!(persisted.bucket_incarnation_id, incarnation);
assert_eq!(persisted.policy_config_json, policy);
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
Ok(())
}
#[tokio::test]
async fn test_bucket_tags_opa_contract() -> TestResult {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
};
init_logging();
let mut opa = OpaMock::start().await?;
let mut env = RustFSTestEnvironment::new().await?;
let opa_url = opa.url.clone();
let server_env = [
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
("NO_PROXY", "127.0.0.1,localhost"),
];
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
])
.await?;
let secret = uuid::Uuid::new_v4().to_string();
create_user(&env, "opabuckettags", &secret).await?;
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -786,7 +733,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let head = client
client
.head_object()
.bucket(source)
.key("report")
@@ -794,8 +741,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
assert_eq!(head.tag_count(), Some(1));
opa.expect_versioned_tag_permission(source, "report", version).await?;
client.list_objects_v2().bucket(source).send().await?;
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
client.list_object_versions().bucket(source).send().await?;
@@ -805,65 +750,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
admin.create_bucket().bucket(destination).send().await?;
set_department(&admin, destination, "engineering").await?;
let copy_source = format!("{source}/report?versionId={version}");
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("copied")
.send()
.await
.expect_err("denied COPY and REPLACE must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
}
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
set_department(&admin, destination, "finance").await?;
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
// The access hook checks preliminary tags, then the copy use case checks
// the exact source reader's tags before publishing the destination.
for _ in 0..2 {
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
}
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
assert_eq!(tags.tag_set().len(), 1);
assert_eq!(tags.tag_set()[0].key(), "department");
assert_eq!(
tags.tag_set()[0].value(),
if directive == TaggingDirective::Copy {
"engineering"
} else {
"legal"
}
);
}
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
let upload = client
.create_multipart_upload()
@@ -939,14 +847,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("denied-source")
.send()
.await
.expect_err("a denied source must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
set_department(&client, source, "finance")
.await
.expect_err("requested tags must not authorize their own mutation");
@@ -985,7 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
.await?;
let invalid_write = admin
admin
.put_bucket_tagging()
.bucket(destination)
.tagging(
@@ -995,25 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.build()?,
)
.send()
.await
.expect_err("new duplicate tags must be rejected before publication");
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
set_department(&admin, source, "finance").await?;
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
let read = client
.get_object()
.bucket(source)
.key("report")
.version_id(version)
.send()
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let error = client
.get_object()
.bucket(destination)
-1
View File
@@ -31,7 +31,6 @@ workspace = true
[features]
default = []
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
# Sampled wall/CPU diagnostics only; does not change the read backend.
diagnostic-local-batch = ["rustix/time"]
+5 -16
View File
@@ -192,7 +192,6 @@ pub mod bucket {
table_catalog_path_hash, unreadable_config_refusal,
};
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
}
pub mod config_parse_mode {
@@ -332,8 +331,8 @@ pub mod bucket {
pub mod utils {
pub use crate::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
is_meta_bucketname, is_valid_object_prefix, serialize,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
serialize,
};
}
@@ -452,15 +451,9 @@ pub mod disk {
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
#[cfg(feature = "test-util")]
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
#[cfg(feature = "test-util")]
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
pub use crate::disk::multipart_index::{
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
MultipartIndexRpcRequest,
};
pub use crate::disk::multipart_source::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -473,9 +466,7 @@ pub mod disk {
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
@@ -626,7 +617,7 @@ pub mod rebalance {
pub mod rio {
pub use crate::io_support::rio::{
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
boxed_reader, compression_metadata_value, wrap_reader,
};
}
@@ -663,9 +654,7 @@ pub mod set_disk {
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::bucket::quota::reservation::{
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
};
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
+5 -325
View File
@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
use crate::bucket::metadata_sys::get_bucket_targets_config;
use crate::bucket::metadata_sys::get_replication_config;
use crate::bucket::remote_s3_client::{
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
UploadResponseDeadline, build_remote_s3_client,
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
};
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
connect_timeout: None,
read_timeout: None,
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
// Replication has no retry budget of its own on the request path,
// so it keeps the SDK's standard three attempts; stating it here
// pins the behaviour to this line instead of an SDK default.
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
health_check_duration: target.health_check_duration,
replicate_sync: target.replication_sync,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::from_env(),
})
}
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
}
}
/// Replication request budgets, snapshotted when a target client is built.
/// Streamed GET bodies are not subject to the upload-response budget.
#[derive(Debug, Clone, Copy)]
pub struct TargetRequestTimeouts {
request: Duration,
complete: Duration,
minimum_transfer_rate: u64,
}
impl Default for TargetRequestTimeouts {
fn default() -> Self {
Self {
request: Duration::from_secs(60),
complete: Duration::from_secs(15 * 60),
minimum_transfer_rate: 1024 * 1024,
}
}
}
impl TargetRequestTimeouts {
fn from_env() -> Self {
let defaults = Self::default();
let seconds = |name: &str, fallback: Duration| {
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
Duration::from_secs(if (1..=86400).contains(&value) {
value
} else {
fallback.as_secs()
})
};
let minimum_transfer_rate =
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
Self {
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
minimum_transfer_rate: if minimum_transfer_rate == 0 {
defaults.minimum_transfer_rate
} else {
minimum_transfer_rate
},
}
}
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
// The SDK can make three attempts. Leave transmission headroom for
// all three; the separate response deadline still bounds a trickling
// target after each upload finishes.
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
RemoteS3RetryPolicy::Disabled => 1,
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
};
let seconds = size
.div_ceil(self.minimum_transfer_rate)
.checked_mul(attempts)
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
let timeout = self
.request
.checked_add(Duration::from_secs(seconds))
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
if Instant::now().checked_add(timeout).is_none() {
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
}
Ok(timeout)
}
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
Ok(aws_sdk_s3::config::Builder::new()
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.upload_timeout(size)?)
.build(),
)
.interceptor(UploadResponseDeadline(self.request)))
}
}
#[derive(Debug)]
pub struct TargetClient {
pub endpoint: String,
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
pub health_check_duration: Duration,
pub replicate_sync: bool,
pub client: Arc<S3Client>,
pub timeouts: TargetRequestTimeouts,
}
impl TargetClient {
@@ -2441,11 +2360,12 @@ impl TargetClient {
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
});
let mut config = self.timeouts.upload_override(size)?;
if forwards_source_checksum {
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
operation = operation.config_override(
aws_sdk_s3::config::Builder::new()
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
);
}
operation = operation.config_override(config);
match operation.send().await {
Ok(output) => {
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2581,7 +2501,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(self.timeouts.upload_override(size)?)
.send()
.await
{
@@ -2619,13 +2538,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(
aws_sdk_s3::config::Builder::new().timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.timeouts.complete)
.build(),
),
)
.send()
.await
{
@@ -3054,241 +2966,11 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_headers,
)
}
#[derive(Debug, Clone)]
struct TimedResponseConnector {
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: bytes::Bytes,
frames: Arc<std::sync::atomic::AtomicUsize>,
dropped: Arc<std::sync::atomic::AtomicUsize>,
}
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for ResponseDrop {
fn drop(&mut self) {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
impl SmithyHttpConnector for TimedResponseConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let settings = self.clone();
HttpConnectorFuture::new(async move {
http_body_util::BodyExt::collect(request.into_body())
.await
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
tokio::time::sleep(settings.upload_delay).await;
let guard = ResponseDrop(Arc::clone(&settings.dropped));
let stream = futures::stream::unfold(
(settings.prefix_bytes, Some(settings.response), true, guard),
move |(remaining, mut body, first, guard)| {
let frames = Arc::clone(&settings.frames);
async move {
if remaining == 0 && body.is_none() {
return None;
}
if !first {
tokio::time::sleep(settings.response_delay).await;
}
let bytes = if remaining > 0 {
bytes::Bytes::from_static(b" ")
} else {
body.take()?
};
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Some((
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
(remaining.saturating_sub(1), body, false, guard),
))
}
},
)
.fuse();
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
let mut response =
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
response.headers_mut().insert("etag", "\"test-etag\"");
Ok(response)
})
}
}
fn timed_response_target(
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: &'static [u8],
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connector = SharedHttpConnector::new(TimedResponseConnector {
upload_delay,
response_delay,
prefix_bytes,
response: bytes::Bytes::from_static(response),
frames: Arc::clone(&frames),
dropped: Arc::clone(&dropped),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let (mut target, _) = header_recording_target_client(Vec::new());
target.timeouts = TargetRequestTimeouts {
request: Duration::from_secs(2),
complete: Duration::from_secs(10),
minimum_transfer_rate: 1024 * 1024,
};
target.client = Arc::new(S3Client::from_conf(
target
.client
.config()
.to_builder()
.http_client(http_client)
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(target.timeouts.request)
.build(),
)
.build(),
));
(target, frames, dropped)
}
#[test]
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
let policy = TargetRequestTimeouts::default();
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
assert_eq!(
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
Duration::from_secs(15420)
);
assert!(policy.upload_timeout(-1).is_err());
let slow = TargetRequestTimeouts {
minimum_transfer_rate: 1,
..policy
};
assert!(slow.upload_timeout(i64::MAX).is_err());
}
#[tokio::test(start_paused = true)]
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
for operation in ["put", "forwarded-checksum", "part"] {
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
let started = tokio::time::Instant::now();
let error = if operation == "part" {
target
.put_object_part(
"bucket",
"key",
"upload",
1,
size,
ByteStream::from(body),
&PutObjectPartOptions::default(),
)
.await
.expect_err("response deadline")
} else {
let mut opts = PutObjectOptions::default();
if operation == "forwarded-checksum" {
opts.user_metadata.insert(
"x-amz-checksum-sha256".to_owned(),
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
);
}
target
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
.await
.expect_err("response deadline")
};
assert!(
started.elapsed() >= Duration::from_secs(6),
"upload gets its full independent window: {error}"
);
assert!(
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
"response must end before the larger total deadline"
);
assert!(
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
"trickle delivered progress before failure"
);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
}
}
#[tokio::test(start_paused = true)]
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
target
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
.await
.expect("slow upload, prompt response");
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
let response = target
.get_object("bucket", "key", None, None, None, HeaderMap::new())
.await
.expect("GET headers");
let started = tokio::time::Instant::now();
let body = response
.body
.collect()
.await
.expect("streamed GET remains outside upload deadline")
.into_bytes();
assert!(started.elapsed() > target.timeouts.request);
assert_eq!(&body[..], b" payload");
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
for prefix in [5, 100] {
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
let started = tokio::time::Instant::now();
let result = target
.complete_multipart_upload(
"bucket",
"key",
"upload",
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
&PutObjectOptions::default(),
)
.await;
if prefix == 5 {
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
} else {
assert!(result.is_err());
assert!(started.elapsed() <= Duration::from_secs(11));
}
assert!(started.elapsed() > target.timeouts.request);
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
#[tokio::test(start_paused = true)]
async fn replication_control_operation_deadline_covers_response_trickle() {
let (target, frames, dropped) =
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
let started = tokio::time::Instant::now();
assert!(target.get_bucket_versioning("bucket").await.is_err());
assert!(started.elapsed() <= Duration::from_secs(3));
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3597,7 +3279,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_uris,
)
@@ -3910,7 +3591,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
})
}
@@ -5487,14 +5487,6 @@ pub async fn put_restore_opts(
if !oi.user_tags.is_empty() {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
}
if let Some(expires) = oi.expires {
meta.insert(
"expires".to_string(),
expires
.format(&time::format_description::well_known::Rfc3339)
.map_err(std::io::Error::other)?,
);
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
metadata_keys::RESTORE.to_string(),
@@ -5514,6 +5506,7 @@ pub async fn put_restore_opts(
// computed MD5 is not the object's public plaintext ETag.
preserve_etag: oi.etag.clone(),
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
//expires: oi.expires,
..Default::default()
})
}
@@ -7151,25 +7144,6 @@ mod tests {
0,
"tier generation lease should release after EOF"
);
let unread = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("open an unread transitioned response");
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
drop(unread);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"canceling an unread response must release the tier lease"
);
}
#[cfg(feature = "test-util")]
-71
View File
@@ -256,38 +256,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
/// Shared tag-entry validation for object requests and persisted bucket tags.
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
fn is_valid_tag_text(value: &str) -> bool {
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
VALID_TAG_TEXT
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
.as_ref()
.is_ok_and(|pattern| pattern.is_match(value))
}
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
for tag in tag_set {
let key = tag
.key
.as_deref()
.filter(|key| !key.is_empty())
.ok_or("Tag key cannot be empty")?;
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
return Err("The TagKey you have provided is invalid");
}
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
return Err("The TagValue you have provided is invalid");
}
if !keys.insert(key) {
return Err("Cannot provide multiple Tags with the same key");
}
}
Ok(())
}
pub const MAX_BUCKET_TAGS: usize = 50;
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -1022,20 +990,6 @@ impl BucketMetadata {
self.encryption_config_updated_at = updated;
}
BUCKET_TAGGING_CONFIG => {
if !data.is_empty() {
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
})?;
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
return Err(Error::InvalidArgument(
self.name.clone(),
String::new(),
"Cannot have more than 50 tags per bucket".to_owned(),
));
}
validate_tag_set_entries(&tagging.tag_set)
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
}
self.tagging_config_xml = data;
// Drop the parsed form (like lifecycle above) so clearing the
// payload can't leave stale parsed tags to be cached.
@@ -1830,31 +1784,6 @@ mod test {
/// backlog#2292: a replicated config is stamped with the source
/// `updated_at` it was given, not the local clock, while the plain
/// `update_config` entry keeps stamping the local clock.
#[test]
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
let mut metadata = BucketMetadata::new("validated-tags");
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
.expect("empty value is valid");
let updated = metadata.tagging_config_updated_at;
for invalid in [
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
] {
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
assert_eq!(metadata.tagging_config_xml, valid);
assert_eq!(metadata.tagging_config_updated_at, updated);
}
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
.expect("deletion repairs legacy corruption");
assert!(metadata.tagging_config_xml.is_empty());
}
#[test]
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
+5 -11
View File
@@ -3428,7 +3428,7 @@ mod tests {
),
(
BUCKET_TAGGING_CONFIG,
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
),
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
(
@@ -4556,13 +4556,8 @@ mod tests {
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
}
let mut bm = BucketMetadata::new(bucket);
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
// Model legacy persisted corruption without passing the write-time validator.
bm.tagging_config_xml = raw.to_vec();
} else {
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
}
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
}
@@ -5743,8 +5738,7 @@ mod tests {
// update only needs one interleaving to show up.
const ROUNDS: usize = 8;
for round in 0..ROUNDS {
let tagging =
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5817,7 +5811,7 @@ mod tests {
let blocked = tokio::spawn({
let node_b = node_b.clone();
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
});
// Long enough for the write to have finished had it not waited: the
@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
// Exercise the real storage entry point before testing the new
// reporting semantics: the same ledger must admit and refuse
// these writes even when a scanner floor claims a million bytes.
if protocol == 1 {
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
assert!(
matches!(
store
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
.await,
Err(StorageError::SlowDown)
),
"a missing authoritative ledger must not admit a write from scanner reporting"
);
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
.await
.expect("bootstrap authoritative usage through the production worker");
}
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
assert!(
matches!(
+49 -739
View File
@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{DiskAPI, error::DiskError};
use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::namespace::NamespaceLocking;
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
use rustfs_lock::NamespaceLockGuard;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashSet};
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use time::OffsetDateTime;
@@ -36,8 +36,6 @@ use tracing::warn;
use uuid::Uuid;
const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
const RECONCILE_RETRY_SECONDS: i64 = 30;
const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
@@ -98,10 +96,6 @@ struct QuotaLedger {
version: u8,
bucket_incarnation: Uuid,
quota_revision_unix_nanos: i128,
/// Changes on every durable write, including zero-growth reservations.
/// Missing on an older writer's output, which invalidates a concurrent scan.
#[serde(default)]
writer_revision: Uuid,
accounted_usage: u64,
reservations: BTreeMap<Uuid, PersistedReservation>,
#[serde(default)]
@@ -116,7 +110,6 @@ impl QuotaLedger {
version: QUOTA_LEDGER_FORMAT_VERSION,
bucket_incarnation,
quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
writer_revision: Uuid::nil(),
accounted_usage,
reservations: BTreeMap::new(),
reconcile_required: false,
@@ -492,23 +485,22 @@ impl QuotaContext {
let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
let mut ledger =
load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
.await?;
let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
let Some(mut ledger) = ledger.take() else {
// Bootstrap has no authoritative usage yet. Release every
// request-owned fence by returning; the worker owns no request.
job.schedule();
return Err(StorageError::SlowDown);
};
let mut ledger = load_current_ledger_locked(
Arc::clone(&store),
&ledger_data.bucket,
&ledger_data.ledger_object,
bucket_incarnation,
quota_revision,
)
.await?;
let growth = reservation.growth();
let current_usage = ledger.admitted_usage()?;
let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
if ledger.reconcile_required
|| (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
{
job.schedule();
let mut current_usage = ledger.admitted_usage()?;
let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
current_usage = ledger.admitted_usage()?;
expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
}
if growth > 0 && expected_usage > quota_limit {
return Err(StorageError::QuotaExceeded {
@@ -526,7 +518,7 @@ impl QuotaContext {
});
}
ledger.reserve(operation_id, reservation)?;
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
Ok(QuotaReservation {
ledger: Some(ReservationLedgerData::Single(ledger_data)),
@@ -1090,7 +1082,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
ledger.mark_commit_started(data.operation_id, &data.reservation)?;
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
})
.await
.map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
@@ -1121,7 +1113,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
} else {
ledger.abort(operation_id, &reservation)?;
}
save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
})
.await
.map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
@@ -1129,15 +1121,22 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
async fn load_current_ledger_locked(
store: Arc<ECStore>,
bucket: &str,
ledger_object: &str,
bucket_incarnation: Uuid,
quota_revision: OffsetDateTime,
) -> Result<Option<QuotaLedger>> {
match load_ledger_locked(store, ledger_object).await {
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
) -> Result<QuotaLedger> {
match load_ledger_locked(Arc::clone(&store), ledger_object).await {
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
let usage = exact_bucket_usage(&store, bucket).await?;
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
}
Ok(_) => Err(StorageError::PartMissingOrCorrupt),
Err(StorageError::ConfigNotFound) => Ok(None),
Err(StorageError::ConfigNotFound) => {
let usage = exact_bucket_usage(&store, bucket).await?;
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
}
Err(err) => Err(err),
}
}
@@ -1246,11 +1245,15 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
for (object, target) in orphan_commit_targets {
fence_namespace_mutations(&store, bucket, &object, target).await?;
}
let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
if ledger.reservations.is_empty() && ledger.reconcile_required {
reconcile_exact(&store, bucket, &mut ledger).await?;
removed = true;
}
if !removed && !cursor_changed {
return Ok(());
}
save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
}
fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
@@ -1270,217 +1273,13 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
Ok(removed)
}
#[derive(Debug, Serialize, Deserialize)]
struct ReconcileControl {
bucket_incarnation: Uuid,
quota_revision_unix_nanos: i128,
next_attempt_at: i64,
}
#[derive(Clone)]
struct ReconcileJob {
store: Arc<ECStore>,
bucket: String,
bucket_incarnation: Uuid,
quota_revision: OffsetDateTime,
}
type ReconcileKey = (Uuid, String, Uuid, i128);
fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
ACTIVE.get_or_init(Default::default)
}
struct ActiveReconciliation(ReconcileKey);
impl Drop for ActiveReconciliation {
fn drop(&mut self) {
active_reconciliations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&self.0);
}
}
impl ReconcileJob {
fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
Self {
store,
bucket,
bucket_incarnation,
quota_revision,
}
}
fn schedule(self) {
let key = (
self.store.id,
self.bucket.clone(),
self.bucket_incarnation,
self.quota_revision.unix_timestamp_nanos(),
);
if !active_reconciliations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(key.clone())
{
return;
}
let owner = ActiveReconciliation(key);
// The owner carries no object, metadata, lifecycle or reservation guard.
// Cancellation of the rejected request cannot cancel its reconciliation.
tokio::spawn(async move {
let _owner = owner;
if let Err(error) = self.run().await {
warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
error = %error, "Quota reconciliation deferred");
}
});
}
fn control_object(&self) -> String {
format!(
"{}.reconcile/{}/{}",
ledger_object(&self.bucket),
self.bucket_incarnation,
self.quota_revision.unix_timestamp_nanos()
)
}
async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
if guard.is_lock_lost() {
return Err(StorageError::SlowDown);
}
let mut opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let _ = opts.set_quota_admission(0, u64::MAX);
opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
}
async fn run(&self) -> Result<()> {
let control_object = self.control_object();
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
return Ok(());
};
let guard = Arc::new(guard);
let now = now_unix();
match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
Ok(bytes) => {
let control: ReconcileControl = serde_json::from_slice(&bytes)?;
if control.bucket_incarnation != self.bucket_incarnation
|| control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
{
return Err(StorageError::PartMissingOrCorrupt);
}
if control.next_attempt_at > now {
return Ok(());
}
}
Err(StorageError::ConfigNotFound) => {}
Err(error) => return Err(error),
}
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
let mut control = ReconcileControl {
bucket_incarnation: self.bucket_incarnation,
quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
};
// Persist the throttle before work, including bootstrap and failed scans.
self.save_control(&control_object, &control, &guard).await?;
let result = self.scan_and_publish(&guard).await;
control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
self.save_control(&control_object, &control, &guard).await?;
result
}
/// Take a short authoritative configuration snapshot without recursively
/// acquiring a reader behind a queued metadata/lifecycle writer.
async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
let guard = snapshot
.metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
.ok_or(StorageError::PreconditionFailed)?;
let opts = ObjectOptions {
expected_bucket_incarnation_id: Some(self.bucket_incarnation),
object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
..Default::default()
};
let (quota, incarnation, revision) =
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
.await?;
if incarnation != self.bucket_incarnation
|| revision != self.quota_revision
|| !quota.as_ref().is_some_and(|quota| {
quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
})
{
return Err(StorageError::PreconditionFailed);
}
Ok(snapshot)
}
async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
let ledger_object = ledger_object(&self.bucket);
let expected_revision = {
// Lock order: reconcile owner, lifecycle, metadata, ledger.
// Request paths never wait for the reconcile owner.
let _configuration = self.configuration_snapshot().await?;
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
Ok(mut ledger) => {
if !ledger.reservations.is_empty() {
return Ok(());
}
// Seed a non-nil token even for a legacy ledger. An old
// node drops this field on write and cannot pass the CAS.
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
Some(ledger.writer_revision)
}
Err(StorageError::ConfigNotFound) => None,
Err(error) => return Err(error),
}
};
#[cfg(test)]
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
// Only the independent reconcile owner is held during this whole-bucket
// walk. Concurrent reserve/settle/reap writes invalidate its result.
let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
#[cfg(test)]
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
let configuration = self.configuration_snapshot().await?;
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
Ok(ledger) => {
expected_revision == Some(ledger.writer_revision)
&& !ledger.writer_revision.is_nil()
&& ledger.reservations.is_empty()
}
Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
Err(error) => return Err(error),
};
if !unchanged {
return Ok(());
}
if owner.is_lock_lost()
|| !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
{
return Err(StorageError::SlowDown);
}
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
if !ledger.reservations.is_empty() {
return Err(StorageError::PartMissingOrCorrupt);
}
ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
ledger.reconcile_required = false;
Ok(())
}
async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
@@ -2136,7 +1935,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
async fn save_ledger_locked(
store: Arc<ECStore>,
ledger_object: &str,
ledger: &mut QuotaLedger,
ledger: &QuotaLedger,
ledger_guard: &Arc<NamespaceLockGuard>,
) -> Result<()> {
if ledger_guard.is_lock_lost() {
@@ -2160,71 +1959,9 @@ async fn save_ledger_locked(
let _ = opts.set_quota_admission(0, u64::MAX);
opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
ledger.writer_revision = Uuid::new_v4();
save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
}
/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
/// The production reaper, ownership fence, scan and compare-and-swap still run.
#[cfg(any(test, feature = "test-util"))]
pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let active = active_reconciliations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains(&key);
if !active {
break;
}
tokio::task::yield_now().await;
}
})
.await
.map_err(|_| StorageError::SlowDown)?;
reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
let object = job.control_object();
{
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
match read_config_no_lock(Arc::clone(&store), &object).await {
Ok(bytes) => {
let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
control.next_attempt_at = 0;
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
job.save_control(&object, &control, &guard).await?;
}
Err(StorageError::ConfigNotFound) => {}
Err(error) => return Err(error),
}
}
job.run().await
}
/// Bootstrap a test bucket through the real background reconciliation path.
/// Fixtures call this after enabling V1 quota, before issuing writes whose
/// contract is independent of the initial retryable admission response.
#[cfg(any(test, feature = "test-util"))]
pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
if !quota
.as_ref()
.is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
{
return Ok(());
}
ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
.run()
.await?;
match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
Some(_) => Ok(()),
None => Err(StorageError::SlowDown),
}
}
#[cfg(any(test, feature = "test-util"))]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn fail_next_quota_ledger_save_for_test() {
@@ -2266,434 +2003,6 @@ mod tests {
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use serial_test::serial;
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum ReconcilePauseStage {
BeforeScan,
BeforePublish,
}
struct ReconcilePause {
bucket: String,
stage: ReconcilePauseStage,
armed: std::sync::atomic::AtomicBool,
scans: std::sync::atomic::AtomicUsize,
fail_scan: std::sync::atomic::AtomicBool,
reached: tokio::sync::Notify,
release: tokio::sync::Notify,
}
static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
struct ReconcilePauseGuard(Arc<ReconcilePause>);
impl ReconcilePauseGuard {
fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
let pause = Arc::new(ReconcilePause {
bucket: bucket.to_owned(),
stage,
armed: true.into(),
scans: 0.into(),
fail_scan: false.into(),
reached: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
});
*RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
Self(pause)
}
async fn reached(&self) {
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
.await
.expect("reconciliation reached barrier");
}
}
impl Drop for ReconcilePauseGuard {
fn drop(&mut self) {
self.0.release.notify_one();
let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
*hook = None;
}
}
}
pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
use std::sync::atomic::Ordering;
let hook = RECONCILE_PAUSE
.lock()
.expect("reconcile hooks")
.as_ref()
.filter(|hook| hook.bucket == bucket)
.cloned();
if let Some(hook) = hook {
if stage == ReconcilePauseStage::BeforeScan {
hook.scans.fetch_add(1, Ordering::SeqCst);
}
if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
hook.reached.notify_one();
hook.release.notified().await;
}
if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
return Err(StorageError::other("injected quota usage failure"));
}
}
Ok(())
}
async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
use crate::object_api::PutObjReader;
use crate::storage_api_contracts::object::ObjectIO;
let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let mut seed = PutObjReader::from_vec(vec![1; 10]);
store
.put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
.await
.expect("existing usage");
store
.update_bucket_metadata_config(
&bucket,
"quota.json",
serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
)
.await
.expect("enable quota");
let (_, incarnation, revision) = store
.bucket_quota_and_incarnation_from_disk(&bucket)
.await
.expect("quota identity");
(directories, ReconcileJob::new(store, bucket, incarnation, revision))
}
async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
.await?
.reserve(old, new)
.await
}
async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
.await
.expect("ledger")
}
async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
let key = (
job.store.id,
job.bucket.clone(),
job.bucket_incarnation,
job.quota_revision.unix_timestamp_nanos(),
);
tokio::time::timeout(Duration::from_secs(10), async {
while active_reconciliations().lock().expect("active workers").contains(&key) {
tokio::task::yield_now().await;
}
})
.await
.expect("owned reconciliation finished");
}
async fn expire_reconciliation_backoff(job: &ReconcileJob) {
drain_reconciliation_scheduler(job).await;
let object = job.control_object();
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
let mut control: ReconcileControl =
serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
.expect("control JSON");
control.next_attempt_at = 0;
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
.await
.expect("control fence");
job.save_control(&object, &control, &guard).await.expect("advance test clock");
}
async fn wait_for_reconcile_owner(job: &ReconcileJob) {
let object = job.control_object();
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
.await
.expect("worker finished")
.expect("owner guard");
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
use std::sync::atomic::Ordering;
let (_directories, job) = reconciliation_fixture(10).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
assert!(
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
"unknown usage cannot admit a reservation"
);
pause.reached().await;
assert!(
matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
),
"bootstrap must not publish zero usage"
);
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
// Bypass the local scheduler, exercising the cross-node ownership boundary.
tokio::time::timeout(Duration::from_secs(2), job.run())
.await
.expect("contender must not wait for a scan")
.expect("another owner is active");
let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
.await
.expect("scan does not retain lifecycle readers")
.expect("lifecycle writer");
let metadata = tokio::time::timeout(
Duration::from_secs(2),
metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
)
.await
.expect("scan does not retain metadata readers")
.expect("metadata writer");
drop(metadata);
drop(lifecycle);
pause.0.release.notify_one();
wait_for_reconcile_owner(&job).await;
let ledger = read_test_ledger(&job).await;
assert_eq!(ledger.accounted_usage, 10);
assert!(ledger.reservations.is_empty());
assert!(!ledger.writer_revision.is_nil());
for _ in 0..3 {
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
job.run().await.expect("persisted cooldown");
}
drain_reconciliation_scheduler(&job).await;
assert_eq!(
pause.0.scans.load(Ordering::SeqCst),
1,
"denied requests cannot repeatedly scan a full bucket"
);
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
let (_directories, job) = reconciliation_fixture(100).await;
job.run().await.expect("bootstrap");
for mutation in ["abort", "commit", "reap", "legacy"] {
expire_reconciliation_backoff(&job).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
let worker = tokio::spawn({
let job = job.clone();
async move { job.run().await }
});
pause.reached().await;
let initial = read_test_ledger(&job).await.writer_revision;
if mutation == "legacy" {
let object = ledger_object(&job.bucket);
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
document.as_object_mut().expect("ledger map").remove("writer_revision");
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
.await
.expect("ledger fence");
let mut opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let _ = opts.set_quota_admission(0, u64::MAX);
opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
save_config_with_opts(
Arc::clone(&job.store),
&object,
serde_json::to_vec(&document).expect("old node JSON"),
&opts,
)
.await
.expect("old node drops new field");
} else {
let mut reservation = quota_reservation(&job, 10, 10)
.await
.expect("zero-growth admission while scanning");
let reserved = read_test_ledger(&job).await.writer_revision;
assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
reservation.mark_commit_started().await.expect("commit marker");
assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
match mutation {
"abort" => reservation.abort().await,
"commit" => reservation.commit().await,
"reap" => {
reservation.defer_after_fence();
reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
.await
.expect("reap orphan");
}
_ => unreachable!(),
}
}
let written = read_test_ledger(&job).await;
assert!(written.reservations.is_empty());
assert_ne!(written.writer_revision, initial);
pause.0.release.notify_one();
worker.await.expect("worker join").expect("stale scan discarded");
let current = read_test_ledger(&job).await;
assert_eq!(
current.writer_revision, written.writer_revision,
"{mutation}: stale usage must not replace a concurrent writer"
);
assert_eq!(current.accounted_usage, written.accounted_usage);
}
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
use std::sync::atomic::Ordering;
let (_directories, job) = reconciliation_fixture(10).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
pause.0.fail_scan.store(true, Ordering::SeqCst);
pause.0.release.notify_one();
assert!(job.run().await.is_err());
assert!(matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
));
job.run().await.expect("failure cooldown");
assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
pause.0.fail_scan.store(false, Ordering::SeqCst);
expire_reconciliation_backoff(&job).await;
job.run().await.expect("recover after failure");
assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
job.store
.delete_object(&job.bucket, "seed", ObjectOptions::default())
.await
.expect("delete stored usage");
assert!(
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
"stale usage remains conservative during cooldown"
);
expire_reconciliation_backoff(&job).await;
job.run().await.expect("refresh deleted usage");
assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
quota_reservation(&job, 0, 1)
.await
.expect("delete must not leave bucket permanently full")
.abort()
.await;
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
let (_directories, job) = reconciliation_fixture(100).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
let caller = tokio::spawn({
let job = job.clone();
async move {
job.schedule();
std::future::pending::<()>().await
}
});
pause.reached().await;
caller.abort();
assert!(caller.await.expect_err("cancel caller").is_cancelled());
job.store
.update_bucket_metadata_config(
&job.bucket,
"quota.json",
serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
)
.await
.expect("quota changes while scanning");
pause.0.release.notify_one();
wait_for_reconcile_owner(&job).await;
assert!(
matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
),
"old config cannot publish bootstrap usage"
);
drop(pause);
let (_, incarnation, revision) = job
.store
.bucket_quota_and_incarnation_from_disk(&job.bucket)
.await
.expect("current configuration");
let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
let caller = tokio::spawn({
let job = current.clone();
async move {
job.schedule();
std::future::pending::<()>().await
}
});
pause.reached().await;
caller.abort();
assert!(caller.await.expect_err("cancel new caller").is_cancelled());
pause.0.release.notify_one();
wait_for_reconcile_owner(&current).await;
assert!(
read_test_ledger(&current).await.matches(incarnation, revision),
"owned scan publishes after its waiter is canceled"
);
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
use crate::storage_api_contracts::bucket::DeleteBucketOptions;
let (_directories, job) = reconciliation_fixture(100).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
let worker = tokio::spawn({
let job = job.clone();
async move { job.run().await }
});
pause.reached().await;
job.store
.delete_object(&job.bucket, "seed", ObjectOptions::default())
.await
.expect("empty old bucket");
job.store
.delete_bucket(&job.bucket, &DeleteBucketOptions::default())
.await
.expect("retire bucket while scan is paused");
job.store
.make_bucket(&job.bucket, &MakeBucketOptions::default())
.await
.expect("replacement bucket");
job.store
.update_bucket_metadata_config(
&job.bucket,
"quota.json",
serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
)
.await
.expect("replacement quota");
pause.0.release.notify_one();
assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
assert!(matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
));
drop(pause);
initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
.await
.expect("replacement can bootstrap independently");
let replacement = read_test_ledger(&job).await;
assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
assert_eq!(replacement.accounted_usage, 0);
}
fn ledger(accounted_usage: u64) -> QuotaLedger {
QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
}
@@ -2781,7 +2090,8 @@ mod tests {
.expect("persist crash-left reservation fixture");
assert!(matches!(
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
.await,
Err(StorageError::PartMissingOrCorrupt)
));
assert!(matches!(
@@ -2799,10 +2109,10 @@ mod tests {
Err(StorageError::ConfigNotFound)
));
let replacement =
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
.await
.expect("new quota admission must not inherit a fresh stale reservation");
assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
assert!(replacement.reservations.is_empty());
}
#[test]
+2 -87
View File
@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
};
use aws_smithy_runtime_api::client::interceptors::Intercept;
use aws_smithy_runtime_api::client::interceptors::context::{
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
};
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
use aws_smithy_runtime_api::client::result::ConnectorError;
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
pub ca_cert_pem: Option<String>,
pub connect_timeout: Option<Duration>,
pub read_timeout: Option<Duration>,
/// Entire SDK call, including non-streaming response bodies and retries.
pub operation_timeout: Option<Duration>,
/// How many wire requests one logical call may cost. Every consumer
/// declares it; see [`RemoteS3RetryPolicy`].
pub retry: RemoteS3RetryPolicy,
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
Ok(builder.build())
}
/// Only installed on non-streaming upload operations. The budget begins at
/// response headers, independently of the potentially long request upload.
#[derive(Debug)]
pub(crate) struct UploadResponseDeadline(pub Duration);
impl Intercept for UploadResponseDeadline {
fn name(&self) -> &'static str {
"RustfsUploadResponseDeadline"
}
fn modify_before_deserialization(
&self,
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
_runtime_components: &RuntimeComponents,
_cfg: &mut ConfigBag,
) -> Result<(), BoxError> {
let response = context.response_mut();
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
inner: Some(inner),
deadline: Box::pin(tokio::time::sleep(self.0)),
});
Ok(())
}
}
struct UploadResponseBody {
inner: Option<SdkBody>,
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
}
impl http_body::Body for UploadResponseBody {
type Data = bytes::Bytes;
type Error = BoxError;
fn poll_frame(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
use std::future::Future as _;
use std::task::Poll;
if self.inner.is_none() {
return Poll::Ready(None);
}
// The absolute response budget never renews on data, empty frames,
// or trailers. A retry gets its own bounded response window.
if self.deadline.as_mut().poll(cx).is_ready() {
self.inner.take();
return Poll::Ready(Some(Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"replication response body deadline exceeded",
)
.into())));
}
let Some(inner) = self.inner.as_mut() else {
return Poll::Ready(None);
};
let result = std::pin::Pin::new(inner).poll_frame(cx);
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
self.inner.take();
}
result
}
fn is_end_stream(&self) -> bool {
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
}
fn size_hint(&self) -> http_body::SizeHint {
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
}
}
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
/// signature stays valid.
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
config_builder = config_builder.http_client(http_client);
}
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
let mut timeouts = TimeoutConfig::builder();
if let Some(connect_timeout) = spec.connect_timeout {
timeouts = timeouts.connect_timeout(connect_timeout);
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
if let Some(read_timeout) = spec.read_timeout {
timeouts = timeouts.read_timeout(read_timeout);
}
if let Some(operation_timeout) = spec.operation_timeout {
timeouts = timeouts.operation_timeout(operation_timeout);
}
config_builder = config_builder.timeout_config(timeouts.build());
}
@@ -753,7 +674,6 @@ mod tests {
ca_cert_pem: None,
connect_timeout: None,
read_timeout: None,
operation_timeout: None,
retry: RemoteS3RetryPolicy::Disabled,
user_agent_suffix: "",
}
@@ -1107,16 +1027,11 @@ mod tests {
spec.user_agent_suffix = "RustFS-Test/0.0";
spec.connect_timeout = Some(Duration::from_secs(5));
spec.read_timeout = Some(Duration::from_secs(5));
spec.operation_timeout = Some(Duration::from_secs(12));
let config = build_remote_s3_config(&spec)
.await
.expect("spec should build")
.http_client(http_client)
.build();
assert_eq!(
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
Some(Duration::from_secs(12))
);
S3Client::from_conf(config)
.head_bucket()
.bucket("bucket")
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
impl ReplicationMetadataStore {
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
@@ -117,10 +116,6 @@ mod tests {
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
"config/replication/mrf.bin.recovery"
);
assert_eq!(
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
"config/replication/mrf.bin.replay"
);
assert_eq!(
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
"config/replication/force-delete.bin"
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
use super::replication_filemeta_boundary::{
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
replication_statuses_map, version_purge_statuses_map,
};
use super::replication_lock_boundary::ReplicationLockTiming;
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
const MRF_PENDING_CAP: usize = 200_000;
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
/// re-attempts every retained entry, so a target that stays down costs one
/// attempt per entry per interval.
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
#[derive(Debug, Default)]
pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
}
/// Acknowledge only the generation read by the recovery leader. Entries appended
/// after that generation are retained as a suffix and replayed on the next pass.
/// after that generation are retained as a suffix and replayed on the next startup.
/// Lock order: recovery leader lock -> MRF journal object lock.
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
Err(EcstoreError::PreconditionFailed)
}
/// Acquires the MRF recovery leader lock for one journal replay pass.
/// Acquires the MRF recovery leader lock for the startup replay.
/// Returns `None` (after logging) when the lock cannot be created or another
/// node is already processing the backlog.
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
}
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
/// On acknowledgement failure the backlog is preserved for the next replay pass and
/// On acknowledgement failure the backlog is preserved for the next startup and
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
}
}
fn unix_millis_now() -> i64 {
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
}
/// Whether a periodic pass may run, given the start time of the last pass on
/// any node. A stamp more than one interval ahead of this node's clock is skew
/// or a bad value, so it does not suppress replay.
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
let Some(last_started_ms) = last_started_ms else {
return true;
};
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
}
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
.await
.ok()?;
std::str::from_utf8(&data).ok()?.trim().parse().ok()
}
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
if let Err(error) = ReplicationConfigStore::save(
storage.clone(),
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
started_ms.to_string().into_bytes(),
)
.await
{
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to record the MRF replay start time"
);
}
}
/// One MRF journal replay pass. The recovery leader reads the journal,
/// re-drives every entry, and acknowledges the replayed prefix; entries that
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
///
/// Every node runs the periodic loop, and the leader lock only serializes
/// passes. With `min_interval`, a pass is skipped when a pass on any node
/// started within that interval, read from the shared replay stamp, so a
/// cluster replays once per interval rather than once per node. The startup
/// pass passes `None` and always runs. A pass that finds entries records its
/// start time. Returns whether the journal was read.
/// Lock order: recovery leader lock -> replay stamp object lock.
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
return false;
};
let started_ms = unix_millis_now();
if let Some(interval) = min_interval
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
{
return false;
}
let Some(entries) = load_mrf_recovery_entries(storage).await else {
return true;
};
if !entries.is_empty() {
write_mrf_replay_stamp(storage, started_ms).await;
}
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
true
}
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
/// one pass at a time across the cluster, including the startup pass, and the
/// shared replay stamp spaces passes from different nodes by `period`.
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
tokio::spawn(async move {
loop {
tokio::time::sleep(period).await;
replay_mrf_journal(&storage, Some(period)).await;
}
})
}
#[derive(Debug, thiserror::Error)]
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Start background tasks
pool.start_mrf_persister().await;
pool.start_mrf_processor().await;
pool.start_mrf_replayer().await;
pool.start_force_delete_processor().await;
pool
@@ -1581,7 +1462,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
// A source generation already queued or being uploaded is not driven a
// A version that is already queued or being uploaded is not driven a
// second time: the scanner heal pass sees it as PENDING until the
// first upload lands and would otherwise re-queue it every cycle
// (backlog#2362). The key is released when the worker finishes, or
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Starts the MRF processor — one-shot at startup.
///
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
/// the pass while the server runs.
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
/// rewrites any entries that could not be admitted for a later startup retry.
async fn start_mrf_processor(&self) {
let storage = self.storage.clone();
let handle = tokio::spawn(async move {
replay_mrf_journal(&storage, None).await;
});
self.task_handles.lock().await.push(handle);
}
/// Starts the runtime MRF replayer — ongoing background task.
///
/// Entries reach the journal while the server runs (worker-queue overflow,
/// delete-marker purge intents whose watch window drained with targets still
/// dirty). Replaying only at startup left them waiting for a restart.
async fn start_mrf_replayer(&self) {
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
let handle = tokio::spawn(async move {
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
return;
};
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
return;
};
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
});
self.task_handles.lock().await.push(handle);
}
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// writes them to the on-disk MRF file every flush interval (default 10s,
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
/// entries accumulate. Runtime writers append under the journal lock; only the
/// recovery leader of a replay pass may remove an acknowledged prefix.
/// startup recovery leader may remove an acknowledged prefix.
async fn start_mrf_persister(&self) {
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
return;
@@ -2404,47 +2314,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
/// Object versions currently queued or being uploaded, keyed by bucket,
/// object name, version and source generation. `queue_replica_task` admits a generation only once
/// object name and version. `queue_replica_task` admits a version only once
/// while it is in flight; the scanner heal pass and MRF replays that arrive
/// in the meantime are `Skipped` instead of driving a second complete upload
/// (backlog#2362). Entries are removed when the worker finishes the task or
/// when no worker accepted it.
#[derive(Debug, Default)]
pub(crate) struct ReplicationInFlight {
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ReplicationInFlightKey {
bucket: String,
object: String,
version_id: Option<uuid::Uuid>,
generation: ReplicationGenerationSnapshot,
}
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
fn from(info: &ReplicateObjectInfo) -> Self {
Self {
bucket: info.bucket.clone(),
object: info.name.clone(),
version_id: info.version_id,
generation: info.replication_generation.clone(),
}
}
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
}
impl ReplicationInFlight {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
/// Claim `ri`; `false` when the same version is already in flight.
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
self.lock().insert(ReplicationInFlightKey::from(ri))
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
}
fn finish(&self, ri: &ReplicateObjectInfo) {
self.lock().remove(&ReplicationInFlightKey::from(ri));
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
}
#[cfg(test)]
@@ -2457,21 +2348,26 @@ impl ReplicationInFlight {
/// including when replication panics.
struct ReplicationInFlightGuard {
in_flight: Arc<ReplicationInFlight>,
key: ReplicationInFlightKey,
key: ReplicateObjectInfo,
}
impl ReplicationInFlightGuard {
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
Self {
in_flight,
key: ReplicationInFlightKey::from(ri),
key: ReplicateObjectInfo {
bucket: ri.bucket.clone(),
name: ri.name.clone(),
version_id: ri.version_id,
..Default::default()
},
}
}
}
impl Drop for ReplicationInFlightGuard {
fn drop(&mut self) {
self.in_flight.lock().remove(&self.key);
self.in_flight.finish(&self.key);
}
}
@@ -3525,8 +3421,6 @@ mod tests {
allow_first_read: Notify,
read_count: AtomicUsize,
delete_storage_calls: AtomicUsize,
delete_error: StdMutex<EcstoreError>,
replay_stamp: StdMutex<Option<Vec<u8>>>,
reported_size: StdMutex<Option<i64>>,
stream_read_bytes: Arc<AtomicUsize>,
write_count: AtomicUsize,
@@ -3594,24 +3488,6 @@ mod tests {
_h: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader, Self::Error> {
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let stamp = self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned")
.clone()
.ok_or(EcstoreError::FileNotFound)?;
return Ok(Self::GetObjectReader {
object_info: ObjectInfo {
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
..Default::default()
},
stream: Box::new(Cursor::new(stamp)),
buffered_body: None,
body_source: Default::default(),
});
}
if !object.ends_with("/.replication/resync.bin")
&& !object.ends_with("config/replication/mrf.bin")
&& !object.ends_with("config/replication/force-delete.bin")
@@ -3675,16 +3551,6 @@ mod tests {
let lock = self.new_ns_lock(bucket, object).await?;
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
};
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let mut stamp = Vec::new();
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
*self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
return Ok(ObjectInfo::default());
}
if opts.http_preconditions.is_some()
&& let Some(replacement) = self
.shared
@@ -3806,12 +3672,7 @@ mod tests {
_opts: Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error> {
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
Err(self
.shared
.delete_error
.lock()
.expect("delete error lock should not be poisoned")
.clone())
Err(EcstoreError::NotImplemented)
}
async fn delete_objects(
@@ -4068,48 +3929,6 @@ mod tests {
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
}
#[tokio::test]
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, mut rx) = mpsc::channel(4);
pool.workers.write().await.push(tx);
let initial = ReplicateObjectInfo {
bucket: "pending-tags-bucket".to_string(),
name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
size: 4096,
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..Default::default()
};
let metadata = ReplicateObjectInfo {
op_type: ReplicationType::Metadata,
user_tags: String::new(),
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..initial.clone()
};
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
assert_eq!(pool.in_flight.len(), 2);
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
assert_eq!(pool.in_flight.len(), 1);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
for expected in [&initial, &metadata] {
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
panic!("expected object replication");
};
assert_eq!(queued.replication_generation, expected.replication_generation);
}
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
assert_eq!(pool.in_flight.len(), 0);
}
#[tokio::test]
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4352,8 +4171,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -4404,103 +4221,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_worker_already_purged_source_completes_without_retry() {
let arn = "arn:test";
for delete_marker_version in [false, true] {
for version_id in [Uuid::new_v4(), Uuid::nil()] {
for error in [
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
EcstoreError::FileVersionNotFound,
EcstoreError::FileNotFound,
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: (!delete_marker_version).then_some(version_id),
delete_marker_version_id: delete_marker_version.then_some(version_id),
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
replicate_delete_with_outcome(delete, storage).await,
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
);
assert_eq!(
shared.delete_storage_calls.load(Ordering::SeqCst),
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
"the worker must lock and attempt writeback, reading a non-null data version first"
);
}
}
}
}
#[tokio::test]
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
let arn = "arn:test";
for (version_purge, target_status, error) in [
(
true,
VersionPurgeStatusType::Pending,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
(
false,
VersionPurgeStatusType::Empty,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: version_purge.then(Uuid::new_v4),
delete_marker: !version_purge,
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
targets: if version_purge {
HashMap::new()
} else {
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
},
purge_targets: if version_purge {
HashMap::from([(arn.to_string(), target_status)])
} else {
HashMap::new()
},
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
!replicate_delete_with_outcome(delete, storage).await,
"unfinished target work, source I/O failures, and marker creation must remain retryable"
);
}
}
#[tokio::test]
async fn delete_worker_preserves_persisted_admission_without_live_config() {
let arn = "arn:test";
@@ -5584,8 +5304,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -6550,108 +6268,6 @@ mod tests {
assert_eq!(retained.target_arns, entry.target_arns);
}
#[tokio::test]
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
let shared = empty_resync_shared_state();
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
// A force-delete intent from a superseded generation replays as
// Skipped, so a pass that reaches it acknowledges it off the journal.
let stale_intent = |object: &str| MrfReplicateEntry {
bucket: "mrf-runtime-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
let journal_drained = || async {
tokio::time::timeout(Duration::from_secs(5), async {
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.is_ok()
};
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
for object in ["first", "second"] {
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
assert!(
journal_drained().await,
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
);
}
handle.abort();
}
#[tokio::test]
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
let shared = empty_resync_shared_state();
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
let interval = Duration::from_secs(1);
let persist_backlog = |object: &str| {
let entry = MrfReplicateEntry {
bucket: "mrf-node-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[entry]).expect("MRF entry should encode");
};
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
persist_backlog("first");
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
// Node B's timer fires within the same interval with a backlog present.
persist_backlog("second");
assert!(
!replay_mrf_journal(&node_b, Some(interval)).await,
"a node must not replay within one interval of another node's pass"
);
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
assert!(
replay_mrf_journal(&node_b, None).await,
"the startup pass should replay regardless of the shared stamp"
);
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
persist_backlog("third");
tokio::time::sleep(interval).await;
assert!(
replay_mrf_journal(&node_a, Some(interval)).await,
"a periodic pass should replay once the interval has elapsed"
);
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
}
#[test]
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
let interval = Duration::from_secs(300);
let now = 1_000_000_000;
assert!(mrf_replay_due(None, now, interval));
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
assert!(
!mrf_replay_due(Some(now + 1_000), now, interval),
"small clock skew should still space passes"
);
assert!(
mrf_replay_due(Some(now + 300_000), now, interval),
"a stamp an interval ahead must not suppress replay"
);
}
#[test]
fn mrf_entry_delete_marker_roundtrip() {
let dm_vid = Uuid::new_v4();
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
true
}
Err(e) => {
// A batch purge or another worker may have removed this version
// already. Remote failures still control the final outcome below.
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
if !source_already_purged {
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
}
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
},
..Default::default()
});
source_already_purged
false
}
};
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
/// If the marker disappears (deleted before or while the replica landed),
/// purge the replicated marker from the targets, retrying failed targets on
/// later rounds. When the window drains with targets still dirty, persist the
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
/// purge intent as a durable MRF entry so the next startup replays it through
/// `purge_stale_delete_marker_targets`.
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
/// Heal callers retry for the full watch window because the startup MRF
/// processor runs before bucket metadata (and thus target clients) finishes
/// initializing — the first pass can see `target_client_missing` and a later
/// round resolves the client; replay passes are serial background work, so
/// round resolves the client; the replay loop is serial and startup-only, so
/// blocking it for up to the window per dirty entry is acceptable. Live
/// callers run on replication workers where a down target would pin a worker
/// for the whole window, so they attempt once and lean on the durable intent
@@ -6609,7 +6604,6 @@ mod tests {
health_check_duration: std::time::Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
timeouts: Default::default(),
})
}
-12
View File
@@ -259,10 +259,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
check_put_object_args(bucket, object)?;
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
}
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
return Err(StorageError::InvalidArgument(
bucket.to_owned(),
@@ -274,14 +270,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
Ok(())
}
/// New public keys must remain representable in an unencoded XML listing.
/// Existing keys still use the read/delete validators so they can be removed.
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
object
.chars()
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
}
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
check_bucket_and_object_names(bucket, object)
}
@@ -224,8 +224,6 @@ pub struct ListPathRawOptions {
pub recursive: bool,
pub incl_deleted: bool,
pub skip_hidden_prefix_check: bool,
pub hidden_entry_budget: usize,
pub pending_prefix: Option<String>,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
@@ -262,8 +260,6 @@ impl Clone for ListPathRawOptions {
recursive: self.recursive,
incl_deleted: self.incl_deleted,
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
hidden_entry_budget: self.hidden_entry_budget,
pending_prefix: self.pending_prefix.clone(),
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
@@ -293,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
recursive: opts.recursive,
incl_deleted: opts.incl_deleted,
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
hidden_entry_budget: opts.hidden_entry_budget,
pending_prefix: opts.pending_prefix.clone(),
report_notfound: opts.report_not_found,
filter_prefix: opts.filter_prefix.clone(),
forward_to: opts.forward_to.clone(),
@@ -307,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
}
/// How a merged raw listing stream ended.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ListPathRawOutcome {
/// At least one producer stopped at its per-disk limit instead of at the
/// end of its namespace, so the closed stream is not proof of exhaustion.
/// Carried in-band by the metacache stream trailer, so remote producers
/// report it as well.
pub limit_reached: bool,
/// Inclusive sorted boundary proved by every reader before a bounded stop.
pub scan_cut: Option<String>,
}
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -730,8 +722,6 @@ async fn list_path_raw_inner(
// Sticky across rounds: once any reader's trailer says its producer
// stopped at the per-disk limit, the merged stream may hide more keys.
let mut limit_reached = false;
let mut scan_cut = None;
let mut last_scanned_key: Option<String> = None;
loop {
let mut current = MetaCacheEntry::default();
@@ -743,7 +733,7 @@ async fn list_path_raw_inner(
// );
if revjob_rx.is_cancelled() {
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
return Ok(ListPathRawOutcome { limit_reached });
}
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -803,12 +793,7 @@ async fn list_path_raw_inner(
continue;
}
// eof
if r.limit_reached() == Some(true) {
limit_reached = true;
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
DiskError::other("bounded listing reader ended without a scan boundary")
})?);
}
limit_reached |= r.limit_reached() == Some(true);
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
@@ -882,7 +867,7 @@ async fn list_path_raw_inner(
*r = MetacacheReader::new(detached_rd);
continue;
}
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
}
};
@@ -941,7 +926,7 @@ async fn list_path_raw_inner(
if let Some(finished_fn) = opts.finished.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -991,14 +976,6 @@ async fn list_path_raw_inner(
return Err(classify_listing_quorum_failure(&failures));
}
// The next head from a bounded reader is unknown. The previous
// globally consumed group is a safe cut; consuming another reader's
// later head would turn this absence of evidence into a skipped key.
// Check quorum failures above before accepting the bounded result.
if limit_reached {
break;
}
// Break if all at EOF or error.
if at_eof + has_err == readers.len() {
if has_err > 0
@@ -1006,7 +983,7 @@ async fn list_path_raw_inner(
{
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -1017,7 +994,6 @@ async fn list_path_raw_inner(
break;
}
last_scanned_key = Some(current.name.clone());
if agree == readers.len() && !opts.preserve_replica_metadata {
for r in readers.iter_mut() {
if !r.skip_current() {
@@ -1029,7 +1005,7 @@ async fn list_path_raw_inner(
// warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
result = agreed_fn(current) => result?
}
// warn!("list_path_raw: agreed_fn done");
@@ -1053,12 +1029,12 @@ async fn list_path_raw_inner(
if let Some(partial_fn) = opts.partial.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
}
}
}
Ok(ListPathRawOutcome { limit_reached, scan_cut })
Ok(ListPathRawOutcome { limit_reached })
});
let merge_started = std::time::Instant::now();
@@ -1223,24 +1199,6 @@ mod tests {
assert!(!options.skip_total_timeout);
}
#[test]
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
let request = ListPathRawOptions {
bucket: "versioned".to_owned(),
hidden_entry_budget: 100,
pending_prefix: Some("pending/".to_owned()),
forward_to: Some("pending/0099".to_owned()),
..Default::default()
};
// Primary and fallback readers both construct their walk from the
// per-job clone, rather than directly from the request options.
let job = request.clone();
let walk = walk_dir_options(&job);
assert_eq!(walk.hidden_entry_budget, 100);
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
}
#[tokio::test]
async fn list_path_raw_empty_disks_returns_read_quorum() {
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1358,7 +1316,6 @@ mod tests {
outcome.limit_reached,
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
);
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
}
#[tokio::test]
+4 -45
View File
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
#[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
global_internode_metrics,
};
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
use rustfs_utils::get_env_bool;
use sha2::Digest as _;
use sha2::Sha256;
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
use tracing::{error, info, warn};
use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -1588,13 +1590,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_tonic_nonce(nonce, signed_at, path)?;
// A body-bound request also carries a usable v2 signature. Consume both
// nonces so stripping the replay-scope headers cannot authorize it again.
if content_sha256 != UNSIGNED_PAYLOAD {
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
}
Ok(())
check_and_record_tonic_nonce(nonce, signed_at, path)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -2585,43 +2581,6 @@ mod tests {
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_consumes_the_body_bound_v2_nonce() {
ensure_test_rpc_secret();
let audience = "node-a:9000";
let path = "/node_service.NodeService/RecoverDeleteTransaction";
let digest = "a".repeat(64);
for v3_first in [true, false] {
let mut headers =
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
.expect("body-bound v2 headers should build");
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
let mut downgraded = headers.clone();
for header in [
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
] {
downgraded.remove(header);
}
let (first, replay) = if v3_first {
(&headers, &downgraded)
} else {
(&downgraded, &headers)
};
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
.expect("the first delivery must be accepted");
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
.expect_err("a mutation cannot replay through the other signature generation");
assert_eq!(error.to_string(), "RPC request replay detected");
}
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
@@ -2014,21 +2014,9 @@ impl PeerRestClient {
}
result
})
.await;
self.settle_bucket_metadata_reload(bucket, result).await
}
async fn settle_bucket_metadata_reload(
&self,
bucket: &str,
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
) -> Result<()> {
match result {
Ok(result) => self.finalize_result(result).await,
// A local operation deadline does not establish that the peer is
// unreachable. In particular, it must not gate IAM revocations.
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
}
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4765,31 +4753,6 @@ mod tests {
assert!(client.offline.load(Ordering::Acquire));
}
#[tokio::test]
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
let client = test_peer_client();
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
let error = client
.settle_bucket_metadata_reload("slow-policy", elapsed)
.await
.expect_err("local budget expires");
assert!(error.to_string().contains("timed out"));
assert!(
!client.offline.load(Ordering::Acquire),
"local work must not suppress subsequent IAM RPCs"
);
client
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
.await
.expect("peer reply");
let error = client
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
.await
.expect_err("real transport failure");
assert!(error.to_string().contains("connection refused"));
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
}
#[tokio::test]
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
let client = test_peer_client();
+320 -310
View File
@@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
MetadataReservationCancelRequest, MetadataReservationRequest,
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -1082,10 +1083,7 @@ impl RemoteDisk {
if payload.len() > operation.max_wire_payload_bytes() {
return Err(Error::other("delete recovery operation exceeds payload limit"));
}
let settle_timeout = if matches!(
&operation,
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
) {
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
Some(crate::disk::get_delete_recovery_timeout()?)
} else {
None
@@ -1640,7 +1638,6 @@ impl RemoteDisk {
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1712,7 +1709,6 @@ impl RemoteDisk {
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1783,7 +1779,6 @@ impl RemoteDisk {
}
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -2015,7 +2010,6 @@ impl RemoteDisk {
let operation_result = operation().await;
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2030,7 +2024,6 @@ impl RemoteDisk {
// Log success; the waiting guard balances every exit path.
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2460,6 +2453,239 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
}
impl RemoteDisk {
/// Prepare a remote token without retrying the RPC. It owns the bucket
/// publication fence but does not enter the object metadata queue yet.
pub async fn reserve_object_metadata(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
) -> Result<Uuid> {
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, false)
.await
}
/// Use the distinct short-lease RPC; an old peer rejects it before allocating a token.
pub(crate) async fn reserve_object_metadata_short(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
) -> Result<Uuid> {
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, true)
.await
}
async fn reserve_object_metadata_with_protocol(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
short: bool,
) -> Result<Uuid> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
self.execute_with_timeout_for_op_and_health_action(
"reserve_object_metadata",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationRequest {
disk: self.endpoint.to_string(),
bucket: bucket.to_owned(),
object: object.to_owned(),
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
.map_err(|_| Error::other("reservation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let result = if short {
client.reserve_object_metadata_short(request).await
} else {
client.reserve_object_metadata(request).await
};
let response = result
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
if response.protocol_version != if short { 3 } else { 2 } {
return Err(DiskError::UnsupportedDisk);
}
Uuid::from_slice(&response.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or(DiskError::Unexpected)
},
timeout.min(Duration::from_secs(5)),
FailureHealthAction::IgnoreFailure,
)
.await
}
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
/// result before allowing any subsequent unprotected mutation.
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
self.update_object_metadata_reservation(bucket, object, token, timeout, false)
.await
}
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
self.update_object_metadata_reservation(bucket, object, token, timeout, true)
.await
}
async fn update_object_metadata_reservation(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: Duration,
renew_only: bool,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
self.execute_with_timeout_for_op_and_health_action(
"activate_object_metadata_reservation",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationActivateRequest {
disk: self.endpoint.to_string(),
bucket: bucket.to_owned(),
object: object.to_owned(),
token: token.as_bytes().to_vec().into(),
renew_only,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
.map_err(|_| Error::other("reservation activation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.activate_object_metadata_reservation(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
Ok(())
},
timeout.min(Duration::from_secs(5)),
FailureHealthAction::IgnoreFailure,
)
.await
}
/// Cancel an unclaimed remote reservation. False cannot prove that an
/// earlier claim did not start a physical mutation.
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
self.execute_with_timeout_for_op(
"cancel_object_metadata_reservation",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationCancelRequest {
disk: self.endpoint.to_string(),
token: token.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.cancel_object_metadata_reservation(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if let Some(error) = response.error {
return Err(error.into());
}
Ok(response.success)
},
get_max_timeout_duration(),
)
.await
}
/// Publish a staged user-object write using a remote one-shot reservation.
/// The caller must retain the bucket lifecycle fence until the tail drains.
pub async fn rename_data_with_metadata_reservation(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
token: Uuid,
) -> Result<RenameDataResp> {
self.execute_with_timeout_for_op(
"rename_reserved_data",
|| async {
let mut client = self.get_client().await?;
let rename = RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_owned(),
src_path: src_path.to_owned(),
file_info: compat_json(fi)?,
dst_volume: dst_volume.to_owned(),
dst_path: dst_path.to_owned(),
file_info_bin: encode_file_info_msgpack(fi)?.into(),
scanner_publication_lease_token: Bytes::new(),
bucket_incarnation_id: Bytes::new(),
};
let mut request = Request::new(ReservedRenameDataRequest {
rename: Some(rename),
token: token.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
.map_err(|_| Error::other("reserved rename request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.rename_reserved_data(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
decode_msgpack_or_json::<RenameDataResp>(
&response.rename_data_resp_bin,
&response.rename_data_resp,
"RenameDataResp",
)
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed(
&self,
@@ -2719,54 +2945,6 @@ impl RemoteDisk {
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
let expected_disk = self
.id
.lock()
.await
.filter(|id| !id.is_nil())
.ok_or(DiskError::DiskNotFound)?;
let query = MultipartIndexRpcRequest::new(bucket, marker);
let endpoint = self.endpoint.to_string();
let canonical = query.canonical_request(&endpoint)?;
let opts = serde_json::to_string(&query)?;
self.execute_with_timeout(
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(DiskInfoRequest {
disk: endpoint.clone(),
opts: opts.clone(),
});
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
return Err(DiskError::FileCorrupt);
}
// An old server ignores the additive query and returns ordinary DiskInfo.
// It cannot satisfy the dedicated reply schema or the fresh challenge.
let reply: MultipartIndexRpcReply =
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
reply.validate(&query, expected_disk)?;
crate::cluster::rpc::verify_tonic_rpc_response_proof(
&query.canonical_response(&endpoint, &reply.page)?,
&reply.proof,
)
.map_err(Error::other)?;
Ok(reply.page)
},
get_drive_disk_info_timeout(),
)
.await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4350,35 +4528,6 @@ impl DiskAPI for RemoteDisk {
}
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
// Older peers reject this variant. Do not retry through the unbounded protocol.
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::DiscoverBounded {
object: object.to_owned(),
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
Err(Error::DeleteRecoveryWorkLimit)
}
Err(error) => Err(error),
}
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -4414,58 +4563,6 @@ impl DiskAPI for RemoteDisk {
_ => Err(Error::other("unexpected delete discovery reply")),
}
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
_namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
match self
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
.await
{
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
Err(Error::DeleteRecoveryWorkLimit)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::InspectBounded {
object: object.to_owned(),
transaction,
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
Err(Error::DeleteRecoveryWorkLimit)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction(
&self,
volume: &str,
@@ -4986,113 +5083,6 @@ mod tests {
}
}
#[tokio::test]
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("test peer must bind");
let directory = tempfile::tempdir().expect("temporary disk");
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk"),
);
disk.make_volume("bucket").await.expect("bucket");
let prefix = directory.path().join("bucket/prefix");
std::fs::create_dir(&prefix).expect("prefix");
for index in 0..64 {
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
}
let fixture = Arc::new(TestBoundedDiscovery {
disk: Arc::clone(&disk),
response_gate: TestFormatReadGate::default(),
});
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
let remote = remote_disk_for_addr(&peer.addr).await;
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
.await
.expect("disk completed bounded scan");
task.abort();
assert!(task.await.expect_err("caller cancelled").is_cancelled());
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
fixture.response_gate.release.cancel();
peer.stop().await;
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
assert!(prefix.join("child-63").is_dir());
}
#[tokio::test]
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
remote
.set_disk_id(Some(Uuid::new_v4()))
.await
.expect("expected disk identity");
assert!(matches!(
remote.list_multipart_index_roots("bucket", None).await,
Err(DiskError::MethodNotAllowed)
));
assert_eq!(
peer.peer.disk_info_calls.load(Ordering::Acquire),
1,
"exercise the actual legacy JSON handler"
);
peer.stop().await;
}
#[tokio::test]
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
assert_eq!(
remote
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
.await
.expect_err("old peer cannot prove a bounded inventory"),
DiskError::DeleteRecoveryWorkLimit
);
let transaction = Uuid::new_v4();
assert_eq!(
remote
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
.await
.expect_err("old peer cannot inspect with a budget"),
DiskError::DeleteRecoveryWorkLimit
);
assert_eq!(
remote
.settle_delete_transaction_bounded(
"bucket",
crate::disk::BoundedDeleteRecoverySettlement {
object: "prefix".to_owned(),
transaction,
expected_fingerprint: [0; 32],
action: DeleteRecoveryAction::PublishCommit,
max_entries: 65536,
},
None
)
.await
.expect_err("old peer cannot settle with a budget"),
DiskError::DeleteRecoveryWorkLimit
);
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods"),
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
);
peer.stop().await;
}
#[tokio::test]
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -5353,6 +5343,84 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
async fn short_metadata_reservation_rejects_old_peer_without_legacy_rpc() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
assert!(matches!(
remote
.reserve_object_metadata_short("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
vec!["/node_service.NodeService/ReserveObjectMetadataShort"],
"an old peer must never receive the longer-lived v2 request"
);
peer.stop().await;
}
#[tokio::test]
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
let token = Uuid::new_v4();
assert!(matches!(
remote
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.renew_object_metadata("bucket", "object", token, Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote.cancel_object_metadata_reservation(token).await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.rename_data_with_metadata_reservation(
crate::disk::RUSTFS_META_TMP_BUCKET,
"staged",
&sample_rename_data_file_info(),
"bucket",
"object",
token,
)
.await,
Err(DiskError::UnsupportedDisk)
));
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
vec![
"/node_service.NodeService/ReserveObjectMetadata",
"/node_service.NodeService/ActivateObjectMetadataReservation",
"/node_service.NodeService/ActivateObjectMetadataReservation",
"/node_service.NodeService/CancelObjectMetadataReservation",
"/node_service.NodeService/RenameReservedData",
],
"reservation operations must never downgrade to legacy methods",
);
peer.stop().await;
}
#[tokio::test]
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -5985,16 +6053,9 @@ mod tests {
format_data: Bytes,
read_all_data: Bytes,
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
unsupported_methods: Arc<StdMutex<Vec<String>>>,
}
#[derive(Debug)]
struct TestBoundedDiscovery {
disk: Arc<crate::disk::local::LocalDisk>,
response_gate: TestFormatReadGate,
}
#[derive(Debug, Default)]
struct TestFormatReadGate {
entered: tokio::sync::Notify,
@@ -6011,7 +6072,6 @@ mod tests {
format_data,
read_all_data,
next_format_read_gate: Arc::default(),
bounded_discovery: Arc::default(),
unsupported_methods: Arc::default(),
}
}
@@ -6111,56 +6171,6 @@ mod tests {
Ok(grpc.unary(method, request).await)
})
}
"/node_service.NodeService/RecoverDeleteTransaction"
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
{
#[derive(Clone)]
struct BoundedSvc(Arc<TestBoundedDiscovery>);
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
let fixture = Arc::clone(&self.0);
Box::pin(async move {
let request = request.into_inner();
let operation: DeleteRecoveryRpcOperation =
rustfs_protos::decode_delete_recovery_payload(&request.operation)
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
return Err(Status::invalid_argument("unexpected fallback"));
};
let page = fixture
.disk
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
.await
.map_err(|_| Status::internal("bounded discovery failed"))?;
// Lose the response only after the real disk operation has completed.
fixture.response_gate.entered.notify_one();
fixture.response_gate.release.cancelled().await;
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
.map_err(|_| Status::internal("encode bounded reply"))?;
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
success: true,
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
result: result.into(),
error: None,
}))
})
}
}
let fixture = self
.peer
.bounded_discovery
.lock()
.expect("bounded fixture")
.clone()
.expect("configured fixture");
Box::pin(async move {
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec);
Ok(grpc.unary(BoundedSvc(fixture), request).await)
})
}
"/node_service.NodeService/ReadAll" => {
#[derive(Clone)]
struct ReadAllSvc(AuthenticatedReadPeer);
+116 -5
View File
@@ -445,10 +445,6 @@ impl RemoteClient {
}
}
});
// A successful lock RPC over a cached or fresh channel proves the peer is
// reachable; reconcile the internode peer-health gauge so it does not stay
// stale when no fresh dial fires (issue #9119).
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
fn admit_late_release(&self) -> bool {
@@ -676,7 +672,6 @@ impl RemoteClient {
with_lock_peer_health(&addr, |health| {
health.last_success = Some(Instant::now());
});
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
if let Some(late) = late {
late(response).await;
@@ -1320,6 +1315,122 @@ impl LockClient for RemoteClient {
Ok(resp.success)
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
// Older nodes refresh normally but return no conversion marker.
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
if !response.success {
return Ok(None);
}
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
Some(":0") => Some(false),
Some(":1") => Some(true),
_ => None,
})
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
use std::path::PathBuf;
use tokio::fs;
use uuid::Uuid;
struct PhysicalFixture {
_directories: Vec<tempfile::TempDir>,
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
);
}
}
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
let fixture = fixture(0).await;
for pool in disks(&fixture.store).await {
for disk in pool {
let object = object_path(&disk);
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
if entry.file_type().await.expect("fixture entry type").is_dir() {
fs::remove_dir_all(entry.path())
.await
.expect("remove task-owned fixture rollback directory");
}
}
}
}
fixture
}
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
let mut all = Vec::new();
for pool in &store.pools {
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
let members = set
.disks
.read()
.await
.iter()
.cloned()
.map(|disk| disk.expect("identity fixture disk must be online"))
.collect::<Vec<_>>();
let mut heads = Vec::new();
for disk in members {
match disk
.pool_meta_native_recovery(Request::InspectIdentity {})
.await
.expect("physical identity inspection")
{
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
_ => panic!("identity inspection must return its physical head"),
}
}
all.push(heads);
}
all
}
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
let fixture = owner_repair_no_backups_fixture().await;
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("fixture must provide valid committed pool metadata"),
};
meta.version = version;
let raw = meta
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
.expect("legacy fixture should encode in its original version");
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
let all = disks(&fixture.store).await;
let now = OffsetDateTime::now_utc();
for pool in &all {
let heads = replicas(&raw, now).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install exact legacy head");
}
}
if stale {
let old = replicas(&raw, now - Duration::seconds(10)).await;
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
.await
.expect("install stale same-raw legacy head");
}
(fixture, raw)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
drop(state);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for read in &reads {
assert!(
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
let mut source_raw = raw.clone();
// A second MessagePack value is readable by legacy decoding but
// must not be copied as part of an unverified repair payload.
source_raw.push(0xc0);
assert!(
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
"ordinary legacy reads must remain compatible with trailing bytes"
);
let all_disks = disks(&fixture.store).await;
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in all_disks[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install readable legacy source with trailing bytes");
}
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await
.expect("legacy semantic selection must remain readable");
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
drop(state);
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"version {version}: no identity shard may change"
);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
for target_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
_ => panic!("initial V1 metadata must decode"),
};
meta.version = POOL_META_VERSION;
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V1 payload must remain valid"),
};
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V2 payload must remain valid"),
};
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
// Pool two follows pool zero; pools zero and one cover both
// orderings and both target formats in separate runs.
let versions = [first_version, other_version, first_version];
let physical = disks(&fixture.store).await;
let timestamp = OffsetDateTime::now_utc();
for (pool_index, pool) in physical.iter().enumerate() {
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
let heads = replicas(raw, timestamp).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
}
}
let before = snapshots(&fixture.store).await;
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await.expect("equivalent legacy pools should be selectable");
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
tokio::time::timeout(std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
.await.expect("mixed legacy heal must terminate")
.expect("equivalent legacy metadata should heal without downgrading the format floor");
drop(state);
let after = snapshots(&fixture.store).await;
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for (pool_index, read) in reads.iter().enumerate() {
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
assert!(matches!(&read.replica,
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
for (index, snapshot) in after[pool_index].iter().enumerate() {
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
.await.expect("every physical shard must match its selected legacy payload");
}
if pool_index != target_pool {
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
}
}
}
}
},
).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for prepare_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid {
meta,
canonical,
revision,
committed: true,
..
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
_ => panic!("baseline V1 metadata must be committed and readable"),
};
let previous = PoolMetaCommittedCandidate {
canonical: legacy_canonical,
meta: legacy_meta.clone(),
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
};
let mut next = legacy_meta;
next.version = POOL_META_GENERATION_VERSION;
next.pools[0].last_update += Duration::seconds(1);
let revision = PoolMetaRevision {
version: POOL_META_GENERATION_VERSION,
cluster_id: fixture.revision.cluster_id,
epoch: fixture.revision.epoch,
generation: 1,
transaction_id: Some(Uuid::new_v4()),
};
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
.expect("V3 PREPARE with V1 predecessor must encode");
assert!(matches!(decode_pool_meta_replica(pending.clone()),
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
let physical = disks(&fixture.store).await;
for (index, disk) in physical[prepare_pool].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install task-owned V3 PREPARE evidence");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[prepare_pool].replica,
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
if raw == &pending && *actual == revision));
let other_pool = 1 - prepare_pool;
assert!(matches!(&reads[other_pool].replica,
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture
.store
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
)
.await
.expect("mixed V1/V3 PREPARE heal must terminate");
drop(state);
assert!(
result.is_err(),
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"pool {prepare_pool}: rejection must not mutate identity evidence"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("current reader must recognize the legacy source"),
};
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source && revision.version == POOL_META_V1_VERSION));
let physical = disks(&fixture.store).await;
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install legacy source on pool zero");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve every original head and backup"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
"the dangerous source bytes must not be copied into the target pool"
);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("safe pool-zero source must decode"),
};
assert!(
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
"the dangerous target must remain semantically equivalent to the safe source"
);
let physical = disks(&fixture.store).await;
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
.await
.expect("install legacy dangerous target head");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy target heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve all physical heads and backups"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
"unsafe target evidence must not be overwritten by the safe source"
);
},
)
.await;
}
+4 -128
View File
@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
#[cfg(test)]
use std::io::Cursor;
use std::io::Write;
use std::path::PathBuf;
@@ -6838,47 +6839,11 @@ impl Drop for PoolMetaTransactionArm {
}
}
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
if data.len() >= 4
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
// The selection already validates the payload. The runtime-only
// flag can only appear in the legacy three-field sequence or map.
&& rmp::decode::read_array_len(&mut &data[4..])
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
.ok()
== Some(3)
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
&& legacy.dont_save
{
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
}
Ok(())
}
// Ordinary legacy reads retain their compatibility behavior, but bytes that
// will be copied to another pool must not contain an unparsed extension.
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
let payload = data
.get(4..)
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
if decoder.get_ref().position() != payload_len {
return Err(Error::other("pool metadata legacy source has trailing bytes"));
}
Ok(())
}
#[derive(Debug)]
struct PoolMetaSelection {
meta: PoolMeta,
revision: PoolMetaRevision,
canonical: Option<Vec<u8>>,
// Legacy semantic comparison canonicalizes to V2; a heal must instead
// preserve verified raw bytes at the highest observed format floor.
legacy_raw: Option<Vec<u8>>,
legacy_has_unsafe_flag: bool,
replica_state: PoolMetaReplicaState,
cas_tokens: Vec<PoolMetaCasToken>,
absent: bool,
@@ -7269,8 +7234,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: PoolMeta::default(),
revision: PoolMetaRevision::legacy(0),
canonical: None,
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair: false,
repair_write_safe: true,
@@ -7325,8 +7288,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: selected.meta.clone(),
revision: selected.revision,
canonical: Some(selected.canonical.clone()),
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -7340,10 +7301,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
// record; divergent backups without it are ambiguous and fail closed.
let legacy_has_unsafe_flag = committed
.iter()
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
let mut floor_raw = None;
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
for (idx, raw, candidate) in committed {
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7357,36 +7314,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
}
} else {
needs_repair |= selected_raw != &raw;
if floor_raw.is_none()
&& candidate.revision.version == observed_version
&& raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(raw);
}
}
} else {
selected = Some((idx, raw, candidate));
}
}
let (_, selected_raw, mut selected) =
let (_, _, mut selected) =
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
if selected.revision.version == observed_version
&& selected_raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(selected_raw);
}
selected.meta.version = observed_version;
selected.revision.version = observed_version;
Ok(PoolMetaSelection {
meta: selected.meta,
revision: selected.revision,
canonical: Some(selected.canonical),
legacy_raw: floor_raw,
legacy_has_unsafe_flag,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -14606,23 +14546,6 @@ impl ECStore {
.canonical
.as_ref()
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
}
if selection.legacy_has_unsafe_flag {
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
}
let repair_data = if selection.revision.is_generation_protocol() {
canonical.clone()
} else {
let raw = selection
.legacy_raw
.clone()
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
ensure_legacy_pool_meta_payload_complete(&raw)?;
raw
};
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
let pool = self
.pools
.get(pool_index)
@@ -14633,10 +14556,10 @@ impl ECStore {
.get(pool_index)
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
let mut arm = state.arm_transaction();
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
.await?;
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
if confirmed != repair_data {
if confirmed != *canonical {
return Err(Error::other("pool metadata native heal readback diverged"));
}
fence.ensure_held()?;
@@ -29559,53 +29482,6 @@ mod tests {
);
}
#[test]
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
let frame = |payload: Vec<u8>| {
let mut data = Vec::new();
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
data.extend_from_slice(&payload);
data
};
let two_fields = PersistedPoolMetaV1 {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
};
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
let data = frame(payload.expect("two-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
for dont_save in [false, true] {
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
dont_save,
};
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
let data = frame(payload.expect("three-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
}
// This guard is only called for already-decoded replicas; malformed
// containers must fail selection rather than becoming repair sources.
let truncated = frame(vec![0x93, 0x01]);
assert!(matches!(
decode_pool_meta_replica(truncated),
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
));
}
#[test]
fn pool_meta_decode_supports_legacy_payload() {
let start_time = OffsetDateTime::now_utc();
File diff suppressed because it is too large Load Diff
+15 -3
View File
@@ -317,9 +317,8 @@ impl Sets {
&self.ctx
}
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
#[cfg(any(test, feature = "test-util"))]
pub async fn stop_endpoint_monitor_for_test(&self) {
#[cfg(test)]
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
if let Some(exit_signal) = &self.exit_signal {
let _ = exit_signal.send(());
// Wait for an in-flight reconnect pass as well as its receiver to exit.
@@ -656,6 +655,19 @@ impl Sets {
.await
}
pub(crate) async fn put_object_with_prepared_reservations(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
self.get_disks_by_key(object)
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
.await
}
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
&self,
bucket: &str,
+298 -222
View File
@@ -55,20 +55,6 @@ use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use uuid::Uuid;
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
/// Completion may mean legacy fallback after a failed scan; callers must assert
/// the expected index result separately. Remote disks have no local worker.
#[cfg(any(test, feature = "test-util"))]
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
match disk.as_ref() {
crate::disk::Disk::Local(local) => {
local.disk.wait_for_multipart_index_bootstrap().await;
Ok(())
}
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
}
}
/// Disk health status constants
const DISK_HEALTH_OK: u32 = 0;
const DISK_HEALTH_FAULTY: u32 = 1;
@@ -618,8 +604,69 @@ impl LocalDiskWrapper {
let super::RenameDataGuards {
external_guard,
namespace_owner,
metadata_reservation,
metadata_reservation_token,
..
} = guards;
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
(Some(reservation), None) => Some(reservation),
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
Ok(reservation) => Some(reservation),
// A missing token may already have been claimed by an earlier
// request that is still publishing; do not claim it was safe.
Err(DiskError::DiskOngoingReq) => {
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
}
Err(error) => return super::RenameDataObservation::not_dispatched(error),
},
(None, None) => None,
};
// A queued reservation must acquire its metadata lease before disk
// admission. Otherwise a waiting background mutation could hold the
// disk permit needed by the earlier lease owner.
let metadata_reservation = match metadata_reservation {
Some(reservation) => Some(reservation.wait().await),
None => None,
};
// The claimed reservation stays in this cancellable future while disk
// admission can wait. Only physical work moves into an owned task.
let prechecked_permit = if metadata_reservation.is_some() {
// Background admission is exclusive. Waiting for it while holding
// this object's lease can deadlock an admitted mutation waiting for
// the same lease. An already owned permit does not need to wait.
if current_disk_io_context().is_some_and(|context| {
context.class == DiskIoClass::Background
&& !context
.permit
.as_ref()
.is_some_and(|permit| permit.is_for(&self.io_admission))
}) {
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
}
let preflight = self
.with_native_resources(async {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.acquire_disk_io_permit().await
})
.await;
match preflight {
Ok(Ok(Some(permit))) => Some(permit),
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
}
} else {
None
};
// The reserved metadata lease must outlive a cancelled RPC waiter.
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
Some(Arc::new(external_guard))
} else {
external_guard
};
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
@@ -638,27 +685,54 @@ impl LocalDiskWrapper {
};
let observed = run_owned_mutation(external_guard, move || async move {
let mut preflight_rejection = None;
let result = operation
.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed = Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
let rename = || async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed = if let Some(reservation) = metadata_reservation {
Box::pin(operation.disk.rename_data_observed_with_reservation(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
(namespace_owner, Some(reservation)),
))
.await
} else {
Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
))
.await
};
preflight_rejection = observed.preflight_rejection;
observed.result
};
let result = if let Some(permit) = prechecked_permit {
with_disk_io_permit(Some(permit), async {
operation
.with_native_resources(operation.track_admitted_disk_operation(
"rename_data",
move |io_permit| async move {
let _io_permit = io_permit;
rename().await
},
Duration::ZERO,
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::Write,
true,
))
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
},
timeout_duration,
)
.await;
.await?
})
.await
} else {
operation
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
.await
};
// Health tracking must observe the real disk error, not an Ok tuple.
Ok(super::RenameDataObservation {
result,
@@ -867,8 +941,6 @@ pub(crate) struct DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
#[cfg(test)]
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
/// Caller API operations currently executing through the disk health wrapper.
api_waiting: AtomicU32,
/// Operations rejected because the disk was unavailable for the caller.
@@ -896,8 +968,6 @@ impl Default for DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex::new(None),
#[cfg(test)]
operation_timeout_override: std::sync::Mutex::new(None),
api_waiting: AtomicU32::new(0),
total_errors_availability: AtomicU64::new(0),
total_errors_timeout: AtomicU64::new(0),
@@ -1591,20 +1661,6 @@ impl Default for DiskHealthTracker {
}
}
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct ScopedDiskOperationTimeout {
metrics: Arc<DiskHealthMetricEpoch>,
previous: Option<Duration>,
}
#[cfg(test)]
impl Drop for ScopedDiskOperationTimeout {
fn drop(&mut self) {
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
}
}
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
/// This is similar to Go's xlStorageDiskIDCheck.
#[derive(Debug, Clone)]
@@ -1630,6 +1686,81 @@ pub struct LocalDiskWrapper {
}
impl LocalDiskWrapper {
/// Reserve the metadata mutation lock for one object on this disk instance.
/// Unclaimed reservations expire after 60 seconds.
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.reserve_object_metadata(bucket, object, timeout).await
}
/// Register an object metadata reservation in FIFO order on this disk.
/// The returned token may be claimed before its physical lease is available.
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.enqueue_object_metadata(bucket, object).await
}
/// Prepare an inert reservation token bound to this disk and object.
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
self.prepare_object_metadata_with_owner(bucket, object, None).await
}
pub(crate) async fn prepare_object_metadata_with_owner(
&self,
bucket: &str,
object: &str,
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
}
pub(crate) async fn prepare_object_metadata_with_owner_ttl(
&self,
bucket: &str,
object: &str,
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
ttl: Duration,
) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
}
/// Refresh an inert token while the caller is still staging the write.
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.renew_object_metadata(bucket, object, token)
}
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.activate_object_metadata(bucket, object, token).await
}
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
self.disk.cancel_object_metadata_reservation(token)
}
/// Create a new LocalDiskWrapper
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
@@ -1946,23 +2077,6 @@ impl LocalDiskWrapper {
self.health_check
}
#[cfg(test)]
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
// The metrics epoch follows this drive across clones and reconnects.
// Its scoped override cannot shorten unrelated healthy drives' I/O.
let previous = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.replace(timeout);
ScopedDiskOperationTimeout {
metrics: Arc::clone(&self.metrics),
previous,
}
}
/// Enable health monitoring after disk creation.
/// Used to defer health checks until after startup format loading completes.
pub fn enable_health_check(&self) {
@@ -2457,7 +2571,6 @@ impl LocalDiskWrapper {
object: &str,
transaction: uuid::Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
self.track_disk_health_mutation(
"settle_delete_transaction",
@@ -2465,7 +2578,7 @@ impl LocalDiskWrapper {
|| {
Box::pin(async {
self.disk
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
})
},
@@ -2481,7 +2594,6 @@ impl LocalDiskWrapper {
transaction: uuid::Uuid,
journal: &crate::disk::DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
self.track_disk_health_mutation(
"settle_delete_transaction",
@@ -2489,14 +2601,7 @@ impl LocalDiskWrapper {
|| {
Box::pin(async {
self.disk
.try_finish_and_finalize_prepared_delete_local_with_budget(
volume,
object,
transaction,
journal,
namespace_owner,
work_budget,
)
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
})
},
@@ -2639,6 +2744,7 @@ impl LocalDiskWrapper {
timeout_duration,
timeout_health_action,
mutation,
false,
))
.await?
}
@@ -2650,6 +2756,7 @@ impl LocalDiskWrapper {
timeout_duration: Duration,
timeout_health_action: TimeoutHealthAction,
mutation: DiskMetricMutation,
identity_prechecked: bool,
) -> Result<T>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
@@ -2671,7 +2778,7 @@ impl LocalDiskWrapper {
}
// Check if disk is stale
if let Err(err) = self.check_disk_stale().await {
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
self.metrics.record_availability_error();
return Err(err);
}
@@ -2696,13 +2803,6 @@ impl LocalDiskWrapper {
}
return result;
}
#[cfg(test)]
let timeout_duration = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
// Execute the operation with timeout
let result = tokio::time::timeout(timeout_duration, operation).await;
@@ -2751,18 +2851,6 @@ impl LocalDiskWrapper {
#[async_trait::async_trait]
impl DiskAPI for LocalDiskWrapper {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.list_multipart_index_roots(bucket, marker).await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
// The local implementation shares this binary's capability. Availability
// and disk identity remain mutation checks and write-quorum inputs.
@@ -2816,62 +2904,6 @@ impl DiskAPI for LocalDiskWrapper {
)
.await
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
self.track_disk_health_with_op(
"discover_delete_transactions",
|| async {
self.disk
.discover_delete_transactions_bounded(volume, object, max_entries)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: uuid::Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
self.track_disk_health_with_op(
"inspect_delete_transaction",
|| async {
self.disk
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
self.track_disk_health_mutation(
"settle_delete_transaction",
DiskMetricMutation::Delete,
|| async {
self.disk
.settle_delete_transaction_bounded(volume, request, namespace_owner)
.await
},
crate::disk::get_delete_recovery_timeout()?,
)
.await
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -3840,11 +3872,7 @@ mod tests {
assert!(futures::poll!(&mut operation).is_pending());
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
assert!(futures::poll!(&mut operation).is_pending());
assert!(
!logs.contents().contains("disk_operation_slow"),
"normal disk latency should not warn: {}",
logs.contents()
);
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
time::advance(Duration::from_millis(1)).await;
assert!(futures::poll!(&mut operation).is_pending());
let warning = logs.contents();
@@ -5718,6 +5746,115 @@ mod tests {
drop(metadata_lease);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
let dir = tempfile::tempdir().expect("disk directory");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let meta = dir.path().join(RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
.await
.expect("write format");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
disk.make_volume("bucket").await.expect("bucket volume");
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
let token = wrapper
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
.await
.expect("reserve key");
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
disk.format_info.write().await.last_check = None;
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
disk.install_format_probe_hook(false, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv_timeout(Duration::from_secs(30));
});
let publisher = Arc::clone(&wrapper);
let rename = tokio::spawn(async move {
publisher
.rename_data_observed_with_guards(
RUSTFS_META_TMP_BUCKET,
"staging",
&FileInfo::default(),
"bucket",
"object",
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), entered_rx)
.await
.expect("disk identity preflight starts")
.expect("preflight hook");
rename.abort();
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
let key = disk.get_object_path("bucket", "object").expect("object path");
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
release_tx.send(()).expect("release disk identity preflight");
if !released_before_io {
drop(
tokio::time::timeout(Duration::from_secs(10), successor)
.await
.expect("preflight finishes"),
);
}
assert!(
released_before_io,
"a cancelled request must not retain the reservation before physical mutation starts"
);
}
#[tokio::test]
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
let dir = tempfile::tempdir().expect("disk directory");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
let token = wrapper
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
.await
.expect("reserve key");
let held = wrapper
.io_admission
.background(&CancellationToken::new())
.await
.expect("background permit");
let fi = FileInfo::default();
let rename = crate::disk::with_background_disk_io(
CancellationToken::new(),
wrapper.rename_data_observed_with_guards(
RUSTFS_META_TMP_BUCKET,
"staging",
&fi,
"bucket",
"object",
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
),
);
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
drop(held);
let observation = result.expect("reserved mutation must not wait while holding the object lease");
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
assert!(observation.rejected_before_publication());
}
#[tokio::test]
async fn delete_versions_counts_faulty_drive_availability_rejection() {
let dir = tempfile::tempdir().expect("temp dir should be created");
@@ -6181,67 +6318,6 @@ mod tests {
assert!(health.offline_duration().is_none());
}
#[tokio::test]
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
let directories = [
tempfile::tempdir().expect("faulted drive"),
tempfile::tempdir().expect("healthy drive"),
];
let mut wrappers = Vec::new();
for directory in &directories {
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
wrappers.push(LocalDiskWrapper::new(disk, false));
}
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
Arc::clone(&wrappers[0].disk),
false,
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
);
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
tokio::time::pause();
let operation = || async {
tokio::time::sleep(Duration::from_secs(2)).await;
Ok(())
};
let (faulted, healthy) = tokio::join!(
reconnected.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
wrappers[1].track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
);
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
healthy.expect("healthy drive keeps its caller's deadline");
let shorter_started = tokio::time::Instant::now();
let shorter = wrappers[0]
.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_millis(500),
TimeoutHealthAction::IgnoreFailure,
)
.await;
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
assert!(shorter_started.elapsed() < Duration::from_secs(1));
wrappers[0]
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
.await
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
drop(scope);
reconnected
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
.await
.expect("dropping the scope restores reconnected callers' deadlines");
}
#[tokio::test]
async fn ignored_timeout_does_not_mark_drive_failure() {
let dir = tempfile::tempdir().expect("temp dir should be created");
-6
View File
@@ -172,8 +172,6 @@ pub enum DiskError {
#[error("guarded batch deletion is not supported by this disk")]
GuardedDeleteUnsupported,
#[error("foreground delete recovery work limit reached")]
DeleteRecoveryWorkLimit,
#[error("erasure write quorum")]
ErasureWriteQuorum,
@@ -772,7 +770,6 @@ impl Clone for DiskError {
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
DiskError::ShortWrite => DiskError::ShortWrite,
@@ -832,7 +829,6 @@ impl DiskError {
DiskError::RemoteClientUnavailable(_) => 0x2B,
DiskError::ConditionalDeleteConflict => 0x2C,
DiskError::GuardedDeleteUnsupported => 0x2D,
DiskError::DeleteRecoveryWorkLimit => 0x2E,
}
}
@@ -883,7 +879,6 @@ impl DiskError {
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
0x2C => Some(DiskError::ConditionalDeleteConflict),
0x2D => Some(DiskError::GuardedDeleteUnsupported),
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
_ => None,
}
}
@@ -1667,7 +1662,6 @@ mod tests {
for (error, code) in [
(DiskError::ConditionalDeleteConflict, 0x2C),
(DiskError::GuardedDeleteUnsupported, 0x2D),
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
] {
assert_eq!(error.to_u32(), code);
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
+2 -464
View File
@@ -5,7 +5,6 @@ use super::{FdCache, FdCacheEntry, FdKey};
use std::fs::File;
use std::os::unix::fs::FileExt;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
FdKey {
@@ -53,11 +52,7 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
let (_directory, file) = fixture();
let cache = FdCache::new();
let entry = Arc::new(FdCacheEntry {
file,
len: 3,
invalidated: AtomicBool::new(false),
});
let entry = Arc::new(FdCacheEntry { file, len: 3 });
let old_generation = cache.generation();
cache.invalidate_exact("v", "part").await;
let current_generation = cache.generation();
@@ -130,7 +125,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
assert!(
cache.get(&key("v", "part", direct)).await.is_some(),
"new entries must survive the earlier volume invalidation"
"new entries must survive the earlier volume predicate"
);
}
cache.clear();
@@ -169,7 +164,6 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
let replacement = Arc::new(FdCacheEntry {
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
len: 11,
invalidated: AtomicBool::new(false),
});
for direct in [false, true] {
let key = key("v", "part", direct);
@@ -183,459 +177,3 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
assert_eq!(fresh.len, 11);
}
}
#[tokio::test]
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
let (_directory, file) = fixture();
let mut cache = FdCache::new();
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
cache.invalidate_under("v", "missing/prefix");
cache.invalidate_volume("v");
cache.invalidate_exact("v", "missing/prefix/part.1").await;
cache.clear();
assert!(
cache.maintenance_task.get().is_none(),
"empty-cache invalidations must not spawn maintenance"
);
// Populate a normal entry and verify matched vs unmatched filter gates directly,
// and ensure unmatched invalidations do not enqueue pending removals.
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
assert_eq!(cache.tracker.live_entries(), 1);
assert!(cache.tracker.may_match_volume("v"));
assert!(!cache.tracker.may_match_volume("other-volume"));
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
cache.invalidate_volume("other-volume");
cache.invalidate_under("v", "obj-2/0198a3f0");
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
assert!(
!cache
.tracker
.has_pending_invalidations
.load(std::sync::atomic::Ordering::SeqCst),
"unmatched invalidations must not enqueue maintenance removals"
);
assert!(cache.get(&cached_key).await.is_some());
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
// to per-volume unindexed_counts without degrading other volumes.
let other_vol_key = key("v2", "shallow/part.1", false);
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
let deep_key = key("v", deep_path, false);
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
assert_eq!(
cache
.tracker
.unindexed_counts
.iter()
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
.sum::<u32>(),
1
);
assert!(
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
"deep key in volume v must not disable prefix filtering in volume v2"
);
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
assert!(cache.get(&deep_key).await.is_none());
cache.invalidate_volume("v2");
assert!(cache.get(&other_vol_key).await.is_none());
// Replacing an invalidated key before maintenance drains pending_invalidations
// must release the old descriptor immediately and keep the replacement live
// even when a strong reference to the old CachedFdEntry shell survives until
// maintenance runs its Arc::ptr_eq check.
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
let old_weak = Arc::downgrade(&old_file);
cache.insert(cached_key.clone(), old_file).await;
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
cache.invalidate_under("v", "obj-1/0198a3f0");
assert!(cache.get(&cached_key).await.is_none());
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
assert!(
old_weak.upgrade().is_none(),
"replacing an entry must retire and drop the previous descriptor immediately"
);
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache
.tracker
.has_pending_invalidations
.load(std::sync::atomic::Ordering::SeqCst)
{
tokio::task::yield_now().await;
}
})
.await
.expect("maintenance should drain pending invalidations");
drop(held_old_shell);
assert!(
cache.get(&cached_key).await.is_some(),
"stale pending invalidation must not evict a newer replacement"
);
assert_eq!(
cache
.tracker
.unindexed_counts
.iter()
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
.sum::<u32>(),
0
);
// Dropping a cache while a live non-invalidated entry is present (even if a
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
// must retire the descriptor and release FdCacheTracker without cycles.
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
let live_drop_weak = Arc::downgrade(&live_drop_file);
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
let shutdown_weak = Arc::downgrade(&shutdown_file);
let tracker_weak = Arc::downgrade(&cache.tracker);
let cloned_moka = cache.cache.clone();
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
cache.invalidate_under("v", "shutdown");
if let Some(handle) = cache.maintenance_task.take() {
handle.abort();
let _ = handle.await;
}
drop(cache);
assert!(
live_drop_weak.upgrade().is_none(),
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
);
assert!(
shutdown_weak.upgrade().is_none(),
"dropping cache when maintenance is aborted must still release descriptors"
);
assert!(
tracker_weak.upgrade().is_none(),
"dropping cache must release FdCacheTracker without reference cycles"
);
drop(cloned_moka);
// Short-TTL expired entries (including mixed expired and fresh entries) must
// be retired by invalidate_under and clear() even though Moka's iterator skips
// expired entries.
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
let expired_weak = Arc::downgrade(&expired_file);
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
let fresh_weak = Arc::downgrade(&fresh_file);
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
ttl_cache.invalidate_under("v", "ttl-obj");
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("invalidate_under must retire both expired and fresh matching descriptors");
assert_eq!(ttl_cache.tracker.live_entries(), 0);
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
ttl_cache
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
.await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
let clear_live_weak = Arc::downgrade(&clear_live_file);
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
ttl_cache.clear();
assert!(
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
"clear() must synchronously retire both TTL-expired and live descriptors"
);
assert_eq!(ttl_cache.tracker.live_entries(), 0);
}
#[tokio::test]
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
use std::collections::HashMap;
use std::sync::atomic::Ordering;
let (_directory, file) = fixture();
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
let paths = [
"a/b",
"a/b/part.1",
"a/b/dir/part.2",
"a/bc",
"a/bc/part.1",
"prefix/obj-0001.bin/0198a3f0/part.1",
"prefix/obj-0002.bin/0198a3f0/part.1",
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
];
let prefixes = [
"a",
"a/",
"a/b",
"a/b/",
"a/b/dir",
"a/bc",
"prefix",
"prefix/obj-0001.bin",
"prefix/obj-0001.bin/0198a3f0",
"d1/d2/d3/d4/d5/c_missing",
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
"nonexistent/prefix",
];
let mut candidate_keys = Vec::new();
for v in volumes {
for p in paths {
for direct in [false, true] {
candidate_keys.push(key(v, p, direct));
}
}
}
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
let mut next_u64 = || {
rng ^= rng << 13;
rng ^= rng >> 7;
rng ^= rng << 17;
rng
};
for step in 0..1000 {
let op = next_u64() % 100;
if op < 45 {
// Insert or replace (either fresh or stale generation)
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
let entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: step,
invalidated: AtomicBool::new(false),
});
let cur_gen = cache.generation();
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
if !stale {
oracle.insert(k, entry);
}
} else if op < 65 {
// Exact invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
let p = paths[(next_u64() as usize) % paths.len()];
cache.invalidate_exact(v, p).await;
oracle.remove(&key(v, p, false));
oracle.remove(&key(v, p, true));
} else if op < 88 {
// Prefix invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
cache.invalidate_under(v, raw_prefix);
let trimmed = raw_prefix.trim_end_matches('/');
oracle.retain(|k, _| {
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
});
} else if op < 97 {
// Volume invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
cache.invalidate_volume(v);
oracle.retain(|k, _| k.volume != v);
} else {
// Full clear
cache.clear();
oracle.clear();
}
// Verify lockstep equivalence against the reference oracle after every operation.
for k in &candidate_keys {
let got = cache.get(k).await;
let expected = oracle.get(k);
match (got, expected) {
(Some(g), Some(e)) => assert!(
Arc::ptr_eq(&g, e),
"step {step}: pointer mismatch for key ({}, {}, {})",
k.volume,
k.path,
k.direct
),
(None, None) => {}
(g, e) => panic!(
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
k.volume,
k.path,
k.direct,
g.is_some(),
e.is_some()
),
}
}
}
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
cache.clear();
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("clear must drain all live entries and pending invalidations");
assert_eq!(cache.tracker.live_entries(), 0);
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
use std::sync::atomic::{AtomicU64, Ordering};
let (_directory, file) = fixture();
let cache = Arc::new(FdCache::new());
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
// at `entry_shards[shard].lock()` after it has passed the initial generation
// check and incremented `volume_counts`, bump generation via an unmatched
// invalidation, and verify the post-`new()` recheck refuses publication and
// preserves the existing entry.
let seam_key = key("v-seam", "obj-1/part.1", false);
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
let stale_gen = cache.generation();
let stale_entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: 999_999,
invalidated: AtomicBool::new(false),
});
let blocked_insert = {
let hold = cache.tracker.entry_shards[entry_shard].lock();
let cache_clone = Arc::clone(&cache);
let seam_key_clone = seam_key.clone();
let task = tokio::spawn(async move {
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
});
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
assert!(
std::time::Instant::now() < deadline,
"insert never reached CachedFdEntry::new with the previous entry still live"
);
std::thread::yield_now();
}
cache.invalidate_under("v-seam", "unmatched-prefix");
drop(hold);
task
};
blocked_insert.await.expect("blocked insert task should complete");
let preserved = cache
.get(&seam_key)
.await
.expect("refused stale insert must not retire a valid previous entry");
assert!(Arc::ptr_eq(&preserved, &initial_entry));
cache.invalidate_volume("v-seam");
let floor_v1 = Arc::new(AtomicU64::new(0));
let floor_v2 = Arc::new(AtomicU64::new(0));
let mut handles = Vec::new();
for t in 0..8usize {
let cache = Arc::clone(&cache);
let file = Arc::clone(&file);
let floor_v1 = Arc::clone(&floor_v1);
let floor_v2 = Arc::clone(&floor_v2);
handles.push(tokio::spawn(async move {
for i in 0..250u64 {
let is_v1 = (t + i as usize).is_multiple_of(2);
let vol = if is_v1 { "v1" } else { "v2" };
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
let k = key(vol, &path, i.is_multiple_of(2));
let g = cache.generation();
let entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: g,
invalidated: AtomicBool::new(false),
});
cache.insert_if_fresh(k.clone(), entry, g).await;
let min_gen = if is_v1 {
floor_v1.load(Ordering::SeqCst)
} else {
floor_v2.load(Ordering::SeqCst)
};
if let Some(hit) = cache.get(&k).await {
assert!(
hit.len >= min_gen,
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
hit.len
);
}
}
}));
}
for t in 0..4usize {
let cache = Arc::clone(&cache);
let floor_v1 = Arc::clone(&floor_v1);
let floor_v2 = Arc::clone(&floor_v2);
handles.push(tokio::spawn(async move {
for i in 0..250u64 {
let is_v1 = (t + i as usize).is_multiple_of(2);
let vol = if is_v1 { "v1" } else { "v2" };
match i % 4 {
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
1 => {
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
cache.invalidate_exact(vol, &p).await;
}
2 => {
let before = cache.generation();
cache.invalidate_volume(vol);
let floor = before + 1;
if is_v1 {
floor_v1.fetch_max(floor, Ordering::SeqCst);
} else {
floor_v2.fetch_max(floor, Ordering::SeqCst);
}
}
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
}
tokio::task::yield_now().await;
}
}));
}
for h in handles {
h.await.expect("worker task should succeed");
}
cache.clear();
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("clear after concurrent stress must drain all live entries");
assert_eq!(cache.tracker.live_entries(), 0);
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
}
File diff suppressed because it is too large Load Diff
+70 -6
View File
@@ -444,12 +444,29 @@ async fn prepare_rename_commit_and_destination_metadata(
#[derive(Debug)]
pub(in crate::disk) struct LocalRenamePreflightRejection(());
impl LocalRenamePreflightRejection {
pub(in crate::disk) fn not_dispatched() -> Self {
Self(())
}
}
#[derive(Default)]
pub(super) struct RenameDataState {
namespace_owner: Option<Arc<dyn Send + Sync>>,
metadata_reservation: Option<os::MetadataMutationReservation>,
preflight_rejection: Option<LocalRenamePreflightRejection>,
}
#[cfg(test)]
impl RenameDataState {
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
Self {
metadata_reservation: Some(reservation),
..Default::default()
}
}
}
// Keep optional native MPU state machines out of ordinary rename frames.
// Construct the future here as well as pinning it, so its temporary stays off
// the caller's poll stack. Call only from the existing native MPU branches.
@@ -486,14 +503,20 @@ impl LocalDisk {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
})?;
crate::hp_guard!("LocalDisk::rename_data");
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
self.multipart_index.record_info(dst_path, &fi).await?;
}
#[cfg(feature = "e2e-test-hooks")]
if fi.is_healing() {
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
}
let mut fi = fi;
// Pre-acquired metadata must never reverse quota or multipart-source
// lock order. These paths need their own reservation protocol.
if state.metadata_reservation.is_some()
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
{
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
@@ -589,7 +612,27 @@ impl LocalDisk {
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
None => state.namespace_owner.take(),
};
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
// A reserved lease is single-use and bound to the exact disk path.
// Keep the independent namespace/source owner alongside it; a pre-held
// mutex cannot be acquired again without deadlocking.
let reservation_fence = state
.metadata_reservation
.as_ref()
.and_then(|reservation| reservation.fence());
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
if multipart_staging.is_some() || quota_fence_claim.is_some() {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
let lease = reservation
.into_lease_for(&root_destination_object_path, self.reservation_instance)
.inspect_err(|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
})?;
Arc::new((lease, metadata_owner))
} else {
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
};
drop(metadata_wait);
#[cfg(unix)]
let parent_receipt = if native_timing {
@@ -603,17 +646,23 @@ impl LocalDisk {
};
#[cfg(unix)]
let mut receipt_mutation = native_timing
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
&self.root,
dst_volume,
&root_destination_object_path,
staging_prefix.as_deref(),
Some(metadata_lease),
Some(metadata_owner),
)
.await;
drop(namespace_wait);
// The volume read lease prevents deletion of this disk's bucket until
// publication finishes. Recheck after its wait, before any mutation.
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
if let Some(claim) = quota_fence_claim {
mutation_lease.attach_external_guard(claim);
}
@@ -2552,8 +2601,23 @@ impl LocalDisk {
dst_path: &str,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> super::super::RenameDataObservation {
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
.await
}
pub(in crate::disk) async fn rename_data_observed_with_reservation(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
) -> super::super::RenameDataObservation {
let (namespace_owner, metadata_reservation) = owners;
let mut state = RenameDataState {
namespace_owner,
metadata_reservation,
..Default::default()
};
let result = self
+37 -537
View File
@@ -5,28 +5,6 @@ use crate::disk::{
};
use sha2::{Digest, Sha256};
pub(super) struct RecoveryOperationScope {
pub(super) budget: Arc<crate::object_api::DeleteRecoveryWorkBudget>,
may_publish_receipt: bool,
receipt_published: std::sync::atomic::AtomicBool,
}
tokio::task_local! {
static RECOVERY_OPERATION: Arc<RecoveryOperationScope>;
}
pub(super) fn recovery_operation_scope() -> Option<Arc<RecoveryOperationScope>> {
RECOVERY_OPERATION.try_with(Arc::clone).ok()
}
impl RecoveryOperationScope {
pub(super) fn receipt_published(&self) {
if self.may_publish_receipt {
self.receipt_published.store(true, std::sync::atomic::Ordering::Release);
}
}
}
pub(super) const PREPARED: &str = "delete-prepared";
pub(super) const TARGET: &str = "delete-target";
pub(super) const FINISHED: &str = "delete-commit-finished";
@@ -43,27 +21,22 @@ struct DiscoverySession {
impl DiscoverySession {
/// Consume one bounded step, returning ownership only when traversal remains.
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
.map(|(transactions, next, _)| (transactions, next))
}
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
fn advance(mut self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
if self.frames.is_empty() {
match std::fs::read_dir(&self.root) {
Ok(entries) => self.frames.push((String::new(), entries)),
Err(err) if err.kind() == ErrorKind::NotFound => {
return Ok((Vec::new(), None, 0));
return Ok((Vec::new(), None));
}
Err(err) => return Err(err),
}
}
let mut transactions = BTreeSet::new();
let mut seen_candidates = HashSet::new();
let mut visited = 0u32;
let mut visited = 0usize;
while !self.frames.is_empty()
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
&& visited < max_entries
&& visited < 4096
{
let (parent, entries) = self
.frames
@@ -183,7 +156,7 @@ impl DiscoverySession {
.map(|(object, transaction)| DeleteTransactionId { object, transaction })
.collect();
let next = (!self.frames.is_empty()).then_some(self);
Ok((transactions, next, visited))
Ok((transactions, next))
}
async fn has_evidence(mut self, disk: &LocalDisk, path: &Path) -> Result<bool> {
@@ -607,41 +580,12 @@ fn explicit_delete_referenced_data_dirs(
Ok(Some(referenced))
}
pub(super) async fn next_recovery_entry(entries: &mut tokio::fs::ReadDir) -> std::io::Result<Option<tokio::fs::DirEntry>> {
if let Some(scope) = recovery_operation_scope() {
scope.budget.reserve(1).map_err(std::io::Error::from)?;
}
entries.next_entry().await
}
pub(super) fn recovery_entries(
mut entries: std::fs::ReadDir,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> impl Iterator<Item = std::io::Result<std::fs::DirEntry>> + '_ {
let mut finished = false;
std::iter::from_fn(move || {
if finished {
return None;
}
if let Some(budget) = budget
&& let Err(error) = budget.reserve(1)
{
finished = true;
return Some(Err(error.into()));
}
let entry = entries.next();
finished = entry.is_none();
entry
})
}
fn explicit_delete_plan_bounded(
fn explicit_delete_plan(
root: &Path,
transaction: Uuid,
original: Option<&FileMeta>,
journal: Option<&DeleteRecoveryJournal>,
current_missing: bool,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> Result<Option<ExplicitDeletePlan>> {
let Some(referenced) = explicit_delete_referenced_data_dirs(original, journal, current_missing)? else {
return Ok(None);
@@ -658,7 +602,7 @@ fn explicit_delete_plan_bounded(
let mut candidates = Vec::new();
match std::fs::read_dir(&rollback) {
Ok(entries) => {
for entry in recovery_entries(entries, budget) {
for entry in entries {
let entry = entry.map_err(to_file_error)?;
let kind = entry.file_type().map_err(to_file_error)?;
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
@@ -690,7 +634,7 @@ fn explicit_delete_plan_bounded(
}
match std::fs::read_dir(root) {
Ok(entries) => {
for entry in recovery_entries(entries, budget) {
for entry in entries {
let entry = entry.map_err(to_file_error)?;
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil()) else {
@@ -714,7 +658,7 @@ fn explicit_delete_plan_bounded(
candidates.dedup();
for dir in candidates {
let mut has_parts = false;
for entry in recovery_entries(std::fs::read_dir(&dir).map_err(to_file_error)?, budget) {
for entry in std::fs::read_dir(&dir).map_err(to_file_error)? {
let entry = entry.map_err(to_file_error)?;
if !entry.file_type().map_err(to_file_error)?.is_file() {
return Ok(None);
@@ -753,27 +697,15 @@ fn canonical_uuid(name: &str) -> Option<Uuid> {
Uuid::parse_str(name).ok().filter(|id| name == id.hyphenated().to_string())
}
pub(super) fn missing_marker_preserves_child_prefix(journal: &DeleteRecoveryJournal) -> bool {
journal.original_digest.is_none()
&& journal.original_versions.is_empty()
&& journal.data_dirs.is_empty()
&& journal.target_digest.is_some()
&& !journal.requests.is_empty()
&& journal
.requests
.iter()
.all(|request| request.deleted && request.version_id.is_some_and(|id| !id.is_nil()))
}
pub(super) fn inspection_with_markers_bounded(
pub(super) fn inspection_with_markers(
root: &Path,
volume: &str,
object: &str,
transaction: Uuid,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> Result<(DeleteRecoveryInspection, bool)> {
let rollback = root.join(transaction.to_string());
let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?;
let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?;
let prepared = regular_bytes(&rollback.join(PREPARED))?;
let target = regular_metadata_bytes(&rollback.join(TARGET))?;
let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?;
@@ -807,20 +739,6 @@ pub(super) fn inspection_with_markers_bounded(
{
return Err(DiskError::FileCorrupt);
}
// A failed creation of a marker over a child named `xl.meta` did not
// replace that directory. Older releases nevertheless left a journal.
// Only this bound, originally absent generation may treat it as absence.
let current_path = root.join(STORAGE_FORMAT_FILE);
let current = if !committed
&& before.is_none()
&& (journal.as_ref().is_some_and(missing_marker_preserves_child_prefix)
|| (journal.is_none() && target.is_none() && receipt_bytes.is_none()))
&& std::fs::symlink_metadata(&current_path).is_ok_and(|metadata| metadata.is_dir())
{
None
} else {
regular_metadata_bytes(&current_path)?
};
let backup_digest = before.as_deref().map(digest);
let current_digest = current.as_deref().map(digest);
let prepared_digest = prepared.as_deref().map(digest);
@@ -863,7 +781,7 @@ pub(super) fn inspection_with_markers_bounded(
Err(err) => return Err(to_file_error(err).into()),
};
if let Some(entries) = entries {
for entry in recovery_entries(entries, budget) {
for entry in entries {
let entry = entry.map_err(to_file_error)?;
let kind = entry.file_type().map_err(to_file_error)?;
if !kind.is_file() && !kind.is_dir() {
@@ -880,7 +798,7 @@ pub(super) fn inspection_with_markers_bounded(
descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len()));
if kind.is_dir() && id.is_some_and(|id| !id.is_nil()) {
has_other_uuid_dir |= id != Some(transaction);
for child in recovery_entries(std::fs::read_dir(entry.path()).map_err(to_file_error)?, budget) {
for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? {
let child = child.map_err(to_file_error)?;
let kind = child.file_type().map_err(to_file_error)?;
if !kind.is_dir() && !kind.is_file() {
@@ -926,7 +844,7 @@ pub(super) fn inspection_with_markers_bounded(
}
if name == transaction.to_string() && kind.is_dir() {
has_staged_dir = true;
for file in recovery_entries(std::fs::read_dir(child.path()).map_err(to_file_error)?, budget) {
for file in std::fs::read_dir(child.path()).map_err(to_file_error)? {
let file = file.map_err(to_file_error)?;
let kind = file.file_type().map_err(to_file_error)?;
let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
@@ -1050,8 +968,7 @@ pub(super) fn inspection_with_markers_bounded(
.is_some_and(|referenced| !referenced.contains(&transaction))
}
DeleteRecoveryState::LegacyAmbiguous | DeleteRecoveryState::NewPrepared => {
explicit_delete_plan_bounded(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none(), budget)?
.is_some()
explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some()
}
_ => false,
};
@@ -1085,17 +1002,21 @@ pub(super) fn inspection_with_markers_bounded(
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
let mut map = PREPARED_DELETE_RECEIPTS.lock();
if map.len() >= 4096
&& let Some(evict_key) = map.keys().next().cloned()
{
map.remove(&evict_key);
}
map.insert(rollback, entry);
}
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
PREPARED_DELETE_RECEIPTS.remove(rollback)
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
}
impl LocalDisk {
@@ -1122,7 +1043,6 @@ impl LocalDisk {
let volume = volume.to_owned();
let object = object.to_owned();
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
let recovery_scope = recovery_operation_scope();
os::run_blocking_namespace_operation(lease, move || {
Ok((|| {
let entries = match std::fs::read_dir(&root) {
@@ -1132,7 +1052,7 @@ impl LocalDisk {
};
let mut predecessors = BTreeSet::new();
let mut visited = 0usize;
for entry in recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())) {
for entry in entries {
visited += 1;
if visited > 65536 {
return Err(DiskError::DiskOngoingReq);
@@ -1285,42 +1205,6 @@ impl LocalDisk {
.map_err(|error| to_file_error(error).into())
}
#[cfg(test)]
pub(crate) fn public_delete_discovery_sessions_for_test(&self) -> usize {
DISCOVERY_SESSIONS
.lock()
.values()
.filter(|session| session.disk_root == self.io_root())
.count()
}
pub(super) async fn delete_recovery_discover_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<super::super::BoundedDeleteRecoveryDiscovery> {
if !(1..=super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES).contains(&max_entries) {
return Err(DiskError::InvalidPath);
}
let path = self.delete_discovery_path(volume, Some(object))?;
let session = self.new_delete_discovery_session(volume, Some(object), &path)?;
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
self.validate_delete_discovery_owner(&session)?;
// The blocking owner drops every directory handle even if the RPC waiter disappears.
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
os::run_blocking_namespace_operation(lease, move || {
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
Ok(super::super::BoundedDeleteRecoveryDiscovery {
transactions: if next.is_none() { transactions } else { Vec::new() },
visited,
complete: next.is_none(),
})
})
.await
.map_err(|error| to_file_error(error).into())
}
pub(super) async fn delete_recovery_discover(
&self,
volume: &str,
@@ -1368,39 +1252,6 @@ impl LocalDisk {
.await
}
pub(super) async fn delete_recovery_inspect_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<super::super::BoundedDeleteRecoveryInspection> {
if !safe_path(object, false)
|| transaction.is_nil()
|| !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&max_entries)
{
return Err(DiskError::InvalidPath);
}
let root = self.get_object_path(volume, object)?;
let owner = os::acquire_metadata_mutation_lease(&root, None).await;
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
let volume = volume.to_owned();
let object = object.to_owned();
os::run_blocking_namespace_operation(lease, move || {
let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
Ok(
inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
super::super::BoundedDeleteRecoveryInspection {
inspection,
visited: max_entries - budget.remaining(),
}
}),
)
})
.await
.map_err(to_file_error)?
}
pub(super) async fn delete_recovery_inspect(
&self,
volume: &str,
@@ -1440,22 +1291,9 @@ impl LocalDisk {
let volume = volume.to_owned();
let object = object.to_owned();
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
let scope = recovery_operation_scope();
// Journal inspections ride the durable-mutation lane (the fused delete
// prepare already does): a bulk delete publishes one transaction per
// key, and the one-worker foreground namespace lane would serialize
// those per-drive publishes behind every other mutation.
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
Ok(inspection_with_markers_bounded(
&root,
&volume,
&object,
transaction,
scope.as_ref().map(|scope| scope.budget.as_ref()),
))
})
.await
.map_err(to_file_error)?
os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
.await
.map_err(to_file_error)?
}
async fn activate_delete_preparation(
@@ -1653,49 +1491,6 @@ impl LocalDisk {
self.activate_delete_preparation(&journal, original, target, owner).await
}
pub(super) async fn delete_recovery_settle_bounded(
&self,
volume: &str,
request: super::super::BoundedDeleteRecoverySettlement,
owner: Option<Arc<dyn Send + Sync>>,
) -> Result<super::super::BoundedDeleteRecoverySettlementReply> {
if !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&request.max_entries) {
return Err(DiskError::InvalidPath);
}
let scope = Arc::new(RecoveryOperationScope {
budget: Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(request.max_entries)),
may_publish_receipt: matches!(
&request.action,
DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. }
),
receipt_published: std::sync::atomic::AtomicBool::new(false),
});
let result = RECOVERY_OPERATION
.scope(
Arc::clone(&scope),
self.delete_recovery_settle(
volume,
&request.object,
request.transaction,
request.expected_fingerprint,
request.action,
owner,
),
)
.await;
let outcome = match result {
Ok(inspection) => super::super::DeleteRecoverySettlementOutcome::Inspected(Box::new(inspection)),
Err(DiskError::DeleteRecoveryWorkLimit) if scope.receipt_published.load(std::sync::atomic::Ordering::Acquire) => {
super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
}
Err(error) => return Err(error),
};
Ok(super::super::BoundedDeleteRecoverySettlementReply {
outcome,
visited: request.max_entries - scope.budget.remaining(),
})
}
pub(crate) async fn delete_recovery_settle(
&self,
volume: &str,
@@ -1894,26 +1689,14 @@ impl LocalDisk {
let plan_root = root.clone();
let missing = observed.current_digest.is_none();
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
let scope = recovery_operation_scope();
// Same durable-mutation lane as the journal inspection above: this
// plan closure runs inside every settle on the bulk-delete publish
// path.
let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
let plan = os::run_blocking_namespace_operation(lease, move || {
Ok((|| {
let before =
regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?
.as_deref()
.map(FileMeta::load)
.transpose()?;
explicit_delete_plan_bounded(
&plan_root,
transaction,
before.as_ref(),
None,
missing,
scope.as_ref().map(|scope| scope.budget.as_ref()),
)?
.ok_or(DiskError::FileCorrupt)
explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt)
})())
})
.await
@@ -2178,7 +1961,6 @@ impl LocalDisk {
let (rollback_buf, root_buf, volume_str, object_str) =
(rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
let publication_root = self.publication_root.clone();
let scope = recovery_operation_scope();
return os::run_blocking_namespace_operation_with_sync(receipt_lease, sync, move || {
let _publication_root = publication_root;
for (path, _) in &file_leases {
@@ -2211,14 +1993,8 @@ impl LocalDisk {
}
}
}
let inspected = inspection_with_markers_bounded(
&root_buf,
&volume_str,
&object_str,
transaction,
scope.as_ref().map(|scope| scope.budget.as_ref()),
)
.map(|(state, _)| state);
let inspected =
inspection_with_markers(&root_buf, &volume_str, &object_str, transaction).map(|(state, _)| state);
drop((file_leases, prune_dirs));
Ok(inspected)
})
@@ -2375,7 +2151,6 @@ impl LocalDisk {
let volume_str = volume.to_owned();
let object_str = object.to_owned();
let recovery_scope = recovery_operation_scope();
let res = os::run_blocking_namespace_operation_with_sync(
lease,
sync,
@@ -2422,9 +2197,7 @@ impl LocalDisk {
let mut predecessors = BTreeSet::new();
let mut sole_transaction = true;
if let Ok(entries) = std::fs::read_dir(&file_path) {
for (visited, entry) in
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
{
for (visited, entry) in entries.enumerate() {
if visited >= 65536 {
return Err(DiskError::DiskOngoingReq);
}
@@ -2637,25 +2410,12 @@ impl LocalDisk {
Ok(Some(()))
}
#[cfg(test)]
pub(crate) async fn try_publish_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
self.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, None)
.await
}
pub(crate) async fn try_publish_prepared_delete_local_with_budget(
&self,
volume: &str,
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) || transaction.is_nil() {
@@ -2734,7 +2494,7 @@ impl LocalDisk {
if sole_transaction {
match std::fs::read_dir(&root) {
Ok(entries) => {
for entry in recovery_entries(entries, work_budget.as_deref()) {
for entry in entries {
let entry = entry?;
let Ok(name) = entry.file_name().into_string() else {
sole_transaction = false;
@@ -2782,7 +2542,6 @@ impl LocalDisk {
.map_err(to_file_error)?
}
#[cfg(test)]
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
&self,
volume: &str,
@@ -2790,26 +2549,6 @@ impl LocalDisk {
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<bool> {
self.try_finish_and_finalize_prepared_delete_local_with_budget(
volume,
object,
transaction,
journal,
namespace_owner,
None,
)
.await
}
pub(crate) async fn try_finish_and_finalize_prepared_delete_local_with_budget(
&self,
volume: &str,
object: &str,
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
match res {
@@ -2876,7 +2615,7 @@ impl LocalDisk {
had_physical_data_dirs = true;
let mut removable = Vec::new();
let mut preserve_dir = false;
for entry in recovery_entries(entries, work_budget.as_deref()) {
for entry in entries {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
@@ -4380,100 +4119,6 @@ mod tests {
);
}
#[tokio::test]
async fn delete_marker_collision_rejects_new_journals_and_recovers_legacy_proof() {
let (temp, disk, _info, original) = fixture().await;
let root = disk.io_get_object_path("recovery", "object").expect("object path");
let staged_child = temp.path().join("child-fixture");
fs::rename(&root, &staged_child).await.expect("move intact object");
fs::create_dir(&root).await.expect("create parent prefix");
fs::rename(staged_child, root.join(STORAGE_FORMAT_FILE))
.await
.expect("seed an independent object named object/xl.meta");
let transaction = Uuid::new_v4();
let marker = FileInfo {
volume: "recovery".to_owned(),
name: "object".to_owned(),
version_id: Some(Uuid::new_v4()),
deleted: true,
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
disk.delete_version_recoverable(
"recovery",
"object",
marker.clone(),
true,
DeleteOptions {
old_data_dir: Some(transaction),
..Default::default()
},
)
.await
.expect_err("a child xl.meta directory cannot be replaced by a marker");
assert!(!root.join(transaction.to_string()).exists(), "rejection must precede journal publication");
// Reproduce the durable evidence written by releases that prepared a
// missing-object marker before attempting to replace the directory.
disk.prepare_delete_recovery(
"recovery",
"object",
None,
&[marker],
None,
DeletePreparationContext {
transaction,
bucket_incarnation: None,
namespace_owner: None,
},
)
.await
.expect("seed legacy prepared marker");
let journal_path = root.join(transaction.to_string()).join(PREPARED);
let journal_bytes = fs::read(&journal_path).await.expect("prepared journal");
let mut incompatible: DeleteRecoveryJournal = serde_json::from_slice(&journal_bytes).expect("journal");
incompatible.original_digest = Some([7; 32]);
fs::write(&journal_path, serde_json::to_vec(&incompatible).expect("serialize journal"))
.await
.expect("write incompatible evidence");
assert!(
disk.inspect_delete_transaction("recovery", "object", transaction)
.await
.is_err(),
"a journal that claims existing metadata cannot ignore a directory collision"
);
fs::write(&journal_path, journal_bytes)
.await
.expect("restore exact legacy proof");
let observed = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("inspect legacy proof");
assert_eq!(observed.state, DeleteRecoveryState::NewPrepared);
let recovered = disk
.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback)
.await
.expect("rollback legacy marker without unlinking the child directory");
assert_eq!(recovered.state, DeleteRecoveryState::Absent);
assert!(!root.join(transaction.to_string()).exists());
assert_eq!(
disk.read_all("recovery", "object/xl.meta/xl.meta")
.await
.expect("child metadata")
.as_ref(),
original
);
assert!(
disk.read_version("", "recovery", "object/xl.meta", "", &Default::default())
.await
.is_ok()
);
assert!(
discover_all_keys(&disk, Some("object")).await.is_empty(),
"no proof may keep replacement heal pending"
);
}
// Data movement copies a delete marker onto a target that holds no
// metadata yet. Rolling back that sub-quorum write removes the marker it
// published, and must not report the backup it never wrote as missing:
@@ -6117,151 +5762,6 @@ mod tests {
);
}
#[tokio::test]
async fn delete_recovery_bounded_inspection_charges_root_and_uuid_children() {
let (_temp, disk, _, _) = fixture().await;
let transaction = Uuid::new_v4();
apply(&disk, transaction).await;
let full = disk
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
.await
.expect("full bounded inspection");
assert!(full.visited > 1, "both root and transaction directory entries are charged");
let error = disk
.inspect_delete_transaction_bounded("recovery", "object", transaction, full.visited - 1)
.await
.expect_err("nested entry exceeds allowance");
assert_eq!(error, DiskError::DeleteRecoveryWorkLimit);
let unchanged = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("unchanged proof");
assert_eq!(full.inspection.fingerprint, unchanged.fingerprint);
}
#[tokio::test]
async fn delete_recovery_receipt_publication_distinguishes_deferred_inspection_from_write_failure() {
for write_fails in [false, true] {
let (_temp, disk, _, _) = fixture().await;
let transaction = Uuid::new_v4();
apply(&disk, transaction).await;
let full = disk
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
.await
.expect("measure preparation work");
let receipt = disk
.io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}"))
.expect("receipt path");
if write_fails {
let destination = receipt.clone();
set_owned_file_before_open(&receipt, move || std::fs::create_dir(&destination).expect("reject receipt rename"));
}
let result = disk
.settle_delete_transaction_bounded(
"recovery",
super::super::super::BoundedDeleteRecoverySettlement {
object: "object".to_owned(),
transaction,
expected_fingerprint: full.inspection.fingerprint,
action: DeleteRecoveryAction::PublishCommit,
max_entries: full.visited,
},
None,
)
.await;
if write_fails {
assert!(result.is_err(), "a failed receipt write never counts as publication");
assert!(receipt.is_dir());
} else {
let result = result.expect("persisted receipt is acknowledged independently of post-write inspection");
assert!(matches!(
result.outcome,
super::super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
));
assert_eq!(result.visited, full.visited);
let state = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("durable receipt");
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
}
}
}
#[tokio::test]
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
let (_temp, disk, _, _) = fixture().await;
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
fs::create_dir_all(&path).await.expect("physical prefix");
for index in 0..64 {
fs::create_dir(path.join(format!("child-{index}")))
.await
.expect("legitimate child");
}
for _ in 0..140 {
let page = disk
.discover_delete_transactions_bounded("recovery", "prefix", 8)
.await
.expect("bounded probe");
assert_eq!(page.visited, 8);
assert!(!page.complete, "an empty partial page cannot establish absence");
assert!(page.transactions.is_empty());
}
let page = disk
.discover_delete_transactions_bounded("recovery", "prefix", 128)
.await
.expect("complete probe");
assert!(page.complete);
assert_eq!(page.visited, 64);
assert!(page.transactions.is_empty());
let transaction = Uuid::new_v4();
apply(&disk, transaction).await;
let before = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("before probe");
let page = disk
.discover_delete_transactions_bounded("recovery", "object", 4096)
.await
.expect("complete evidence");
assert!(page.complete);
assert!(page.transactions.iter().any(|id| id.transaction == transaction));
let after = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("after probe");
assert_eq!(before.fingerprint, after.fingerprint);
assert!(
DISCOVERY_SESSIONS
.lock()
.values()
.all(|session| session.disk_root != disk.io_root())
);
}
#[tokio::test]
async fn delete_recovery_cancelled_bounded_discovery_drops_private_session() {
let (_temp, disk, _, _) = fixture().await;
let path = disk.delete_discovery_path("recovery", Some("object")).expect("probe scope");
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
let mut probe = Box::pin(disk.discover_delete_transactions_bounded("recovery", "object", 4096));
assert!(futures::poll!(probe.as_mut()).is_pending());
drop(probe);
assert!(
DISCOVERY_SESSIONS
.lock()
.values()
.all(|session| session.disk_root != disk.io_root())
);
drop(lease);
assert!(
disk.discover_delete_transactions_bounded("recovery", "object", 4096)
.await
.expect("subsequent probe")
.complete
);
}
#[tokio::test]
async fn delete_recovery_evidence_probes_do_not_register_public_cursors() {
let (_temp, disk, _, _) = fixture().await;
File diff suppressed because it is too large Load Diff
@@ -117,7 +117,6 @@ pub(super) struct LoadedSource {
}
pub(super) struct SourceStore {
multipart_index: Arc<super::multipart_index::IndexOwner>,
pub disk_id: Uuid,
pub physical_slot: usize,
pub disk_ids: Vec<Uuid>,
@@ -187,7 +186,6 @@ impl LocalDisk {
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
Ok(SourceStore {
multipart_index: self.multipart_index.clone(),
disk_id,
physical_slot,
disk_ids,
@@ -272,17 +270,21 @@ struct ActiveMultipartSourceEntry {
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
}
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
let source = source.as_ref();
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
return;
};
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
if map.len() >= 4096
&& let Some(evict_key) = map.keys().next().cloned()
{
map.remove(&evict_key);
}
map.insert(
source,
ActiveMultipartSourceEntry {
raw,
@@ -302,50 +304,56 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
}
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
let (raw, fi, head_stamp) =
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
let (raw, fi, head_stamp) = {
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let entry = map.get(source)?;
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return None;
}
Some((raw, fi))
}
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
return false;
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return false;
}
true
}
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
let Some((head_stamp, clean)) =
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
.lock()
.get(source)
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
else {
return false;
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return false;
}
clean
}
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
entry.unsettled_parts.insert(part_number);
entry.parts.remove(&part_number);
});
}
}
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
.with_get(source, |entry| entry.fi.data_dir)
.flatten()
.lock()
.get(source)
.and_then(|entry| entry.fi.data_dir)
.filter(|dir| !dir.is_nil())
else {
return;
@@ -354,9 +362,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
return;
};
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
entry.parts.insert(part.number, (part_stamp, part));
});
}
}
pub(super) fn finish_active_multipart_part_transaction(
@@ -365,29 +373,34 @@ pub(super) fn finish_active_multipart_part_transaction(
committed: bool,
retired_existing_txn: bool,
) {
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let Some(part_number) = part_number.filter(|_| committed) else {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
map.remove(source);
return;
};
if !retired_existing_txn {
return;
}
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
if let Some(entry) = map.get_mut(source)
&& !entry.unsettled_parts.remove(&part_number)
{
map.remove(source);
}
}
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
.with_get(source, |entry| {
if !entry.unsettled_parts.is_empty() {
return None;
}
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
let (expected_stamp, part) = entry.parts.get(&part_number)?;
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
})
.flatten()?;
let (head_stamp, data_dir, expected_stamp, part) = {
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let entry = map.get(source)?;
if !entry.unsettled_parts.is_empty() {
return None;
}
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
let (expected_stamp, part) = entry.parts.get(&part_number)?;
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return None;
}
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
@@ -398,11 +411,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
}
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
}
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
}
/// Canonical source mutations are admitted under SourceRW at the syscall,
@@ -757,9 +770,6 @@ impl SourceStore {
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
}
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
self.multipart_index
.record_head(&format!("{}/xl.meta", self.path), bytes)
.await?;
invalidate_active_multipart_source_head(&self.source);
let path = self.source.join(STORAGE_FORMAT_FILE);
let base = self.base.clone();
@@ -1926,7 +1936,6 @@ impl NativeMultipartPublication {
*self.validated_prepared.lock() = Some(validated);
Ok(())
}
#[cfg(target_os = "linux")]
pub(super) fn prepare_and_install_head_on_thread(
&self,
new_metadata: &[u8],
+116 -182
View File
@@ -27,7 +27,6 @@ mod io_admission;
#[cfg(test)]
pub(crate) mod lifecycle_test_io;
pub mod local;
pub mod multipart_index;
pub mod multipart_source;
pub mod os;
@@ -107,6 +106,8 @@ pub(crate) struct RenameDataGuards {
pub(crate) scanner_publication_lease_token: Option<Uuid>,
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
pub(crate) metadata_reservation_token: Option<Uuid>,
}
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
@@ -123,6 +124,13 @@ impl RenameDataObservation {
}
}
fn not_dispatched(error: DiskError) -> Self {
Self {
result: Err(error),
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
}
}
pub(crate) fn rejected_before_publication(&self) -> bool {
self.result.is_err() && self.preflight_rejection.is_some()
}
@@ -312,6 +320,82 @@ pub enum Disk {
}
impl Disk {
/// Prepare an inert single-use reservation on this disk.
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
pub async fn prepare_object_metadata(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> Result<Uuid> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.prepare_object_metadata(bucket, object),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
}
}
/// Refresh an inert token while its sender stages the write.
pub async fn renew_object_metadata(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: std::time::Duration,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.renew_object_metadata(bucket, object, token),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await,
}
}
/// Enqueue a prepared token in object mutation order.
pub async fn activate_object_metadata(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: std::time::Duration,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.activate_object_metadata(bucket, object, token),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
}
}
/// Cancel an unclaimed reservation. False does not prove that a claim has
/// not already started a physical mutation.
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
match self {
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
}
}
/// Physical ownership belongs to the node hosting the local attachment.
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
match self {
@@ -415,11 +499,10 @@ impl Disk {
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
match self {
Self::Local(disk) => {
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
}
Self::Remote(_) => Ok(None),
@@ -433,19 +516,11 @@ impl Disk {
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
match self {
Self::Local(disk) => {
disk.try_finish_and_finalize_prepared_delete_local(
volume,
object,
transaction,
journal,
namespace_owner,
work_budget,
)
.await
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
}
Self::Remote(_) => Ok(false),
}
@@ -522,17 +597,6 @@ impl Disk {
#[async_trait::async_trait]
impl DiskAPI for Disk {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<multipart_index::MultipartIndexPage> {
match self {
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
}
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
match self {
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
@@ -576,17 +640,6 @@ impl DiskAPI for Disk {
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
}
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<BoundedDeleteRecoveryDiscovery> {
match self {
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
}
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -599,35 +652,6 @@ impl DiskAPI for Disk {
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
}
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: BoundedDeleteRecoverySettlement,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<BoundedDeleteRecoverySettlementReply> {
match self {
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
}
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<BoundedDeleteRecoveryInspection> {
match self {
Self::Local(disk) => {
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
}
Self::Remote(disk) => {
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
}
}
}
async fn inspect_delete_transaction(
&self,
volume: &str,
@@ -1312,18 +1336,33 @@ impl Disk {
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
.await
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
guards.scanner_publication_lease_token,
)
.await,
),
Disk::Remote(remote_disk) => {
if guards.metadata_reservation.is_some() {
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
}
if let Some(token) = guards.metadata_reservation_token {
if guards.scanner_publication_lease_token.is_some() {
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
}
return RenameDataObservation::unknown(
remote_disk
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
.await,
);
}
RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
guards.scanner_publication_lease_token,
)
.await,
)
}
}
}
@@ -1574,15 +1613,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
fn host_name(&self) -> String;
fn endpoint(&self) -> Endpoint;
async fn close(&self) -> Result<()>;
/// Return candidates only while this disk maintains complete boot-local coverage.
/// Unsupported or unready disks must never return an authoritative empty page.
async fn list_multipart_index_roots(
&self,
_bucket: &str,
_marker: Option<&str>,
) -> Result<multipart_index::MultipartIndexPage> {
Err(DiskError::MethodNotAllowed)
}
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
@@ -1760,33 +1790,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
) -> Result<DeleteRecoveryPage> {
Err(DiskError::MethodNotAllowed)
}
/// Bounded foreground inventory owns all traversal state until return or cancellation.
/// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
async fn discover_delete_transactions_bounded(
&self,
_volume: &str,
_object: &str,
_max_entries: u32,
) -> Result<BoundedDeleteRecoveryDiscovery> {
Err(DiskError::DeleteRecoveryWorkLimit)
}
async fn inspect_delete_transaction_bounded(
&self,
_volume: &str,
_object: &str,
_transaction: Uuid,
_max_entries: u32,
) -> Result<BoundedDeleteRecoveryInspection> {
Err(DiskError::DeleteRecoveryWorkLimit)
}
async fn settle_delete_transaction_bounded(
&self,
_volume: &str,
_request: BoundedDeleteRecoverySettlement,
_namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<BoundedDeleteRecoverySettlementReply> {
Err(DiskError::DeleteRecoveryWorkLimit)
}
/// A read-only capability check; unsupported implementations must fail closed.
async fn supports_guarded_delete_versions(&self) -> Result<()> {
Err(DiskError::GuardedDeleteUnsupported)
@@ -2172,13 +2175,6 @@ pub struct WalkDirOptions {
#[serde(default)]
pub skip_hidden_prefix_check: bool,
/// Bound hidden metadata read while proving a non-recursive prefix visible.
#[serde(default)]
pub hidden_entry_budget: usize,
/// A prefix whose visibility was not established before the preceding cut.
#[serde(default)]
pub pending_prefix: Option<String>,
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
pub report_notfound: bool,
@@ -2511,62 +2507,9 @@ pub enum DeleteRecoveryAction {
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
}
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoveryDiscovery {
pub transactions: Vec<DeleteTransactionId>,
pub visited: u32,
pub complete: bool,
}
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoverySettlement {
pub object: String,
pub transaction: Uuid,
pub expected_fingerprint: DeleteRecoveryDigest,
pub action: DeleteRecoveryAction,
pub max_entries: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub enum DeleteRecoverySettlementOutcome {
Inspected(Box<DeleteRecoveryInspection>),
ReceiptPublishedInspectionDeferred,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoverySettlementReply {
pub outcome: DeleteRecoverySettlementOutcome,
pub visited: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoveryInspection {
pub inspection: DeleteRecoveryInspection,
pub visited: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub enum DeleteRecoveryRpcOperation {
SettleBounded(BoundedDeleteRecoverySettlement),
InspectBounded {
object: String,
transaction: Uuid,
max_entries: u32,
},
DiscoverBounded {
object: String,
max_entries: u32,
},
InspectOrphanDataDir {
path: String,
},
@@ -2604,11 +2547,7 @@ impl DeleteRecoveryRpcOperation {
/// Full metadata application and adoption share the existing mutation envelope.
pub fn max_wire_payload_bytes(&self) -> usize {
match self {
Self::SettleBounded(BoundedDeleteRecoverySettlement {
action: DeleteRecoveryAction::AdoptCommit { .. },
..
})
| Self::ApplyVersion { .. }
Self::ApplyVersion { .. }
| Self::ApplyVersions { .. }
| Self::Settle {
action: DeleteRecoveryAction::AdoptCommit { .. },
@@ -2624,11 +2563,8 @@ impl DeleteRecoveryRpcOperation {
pub enum DeleteRecoveryRpcReply {
OrphanDataDir { plain: bool },
Discovered(DeleteRecoveryPage),
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
Inspected(DeleteRecoveryInspection),
InspectedBounded(BoundedDeleteRecoveryInspection),
Settled(DeleteRecoveryInspection),
SettledBounded(BoundedDeleteRecoverySettlementReply),
AppliedVersion,
AppliedVersions(Vec<Option<u32>>),
Evidence { present: bool },
@@ -3106,8 +3042,6 @@ mod tests {
recursive: true,
incl_deleted: false,
skip_hidden_prefix_check: false,
hidden_entry_budget: 0,
pending_prefix: None,
report_notfound: false,
filter_prefix: Some("prefix_".to_string()),
forward_to: Some("object/path".to_string()),
-275
View File
@@ -1,275 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! A candidate index never replaces multipart metadata or quorum validation.
use serde::{Deserialize, Serialize};
use uuid::Uuid;
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
/// A bounded page from one disk's continuously maintained, boot-local index.
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
/// pages from different epochs or physical disk identities.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexPage {
pub bucket: String,
pub disk_id: Uuid,
pub epoch: Uuid,
pub roots: Vec<String>,
pub next_marker: Option<String>,
}
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
/// Old peers may ignore this field, but their ordinary response cannot decode
/// as the required signed, challenge-bound MultipartIndexRpcReply.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexRpcRequest {
pub disk_id: String,
pub metrics: bool,
pub noop: bool,
pub multipart_index: MultipartIndexQuery,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexQuery {
pub operation: String,
pub version: u32,
pub challenge: Uuid,
pub bucket: String,
pub marker: Option<String>,
pub page_cap: usize,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexRpcReply {
pub operation: String,
pub version: u32,
pub challenge: Uuid,
pub page: MultipartIndexPage,
pub proof: Vec<u8>,
}
impl MultipartIndexRpcRequest {
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
Self {
disk_id: String::new(),
metrics: false,
noop: true,
multipart_index: MultipartIndexQuery {
operation: "list_multipart_index_roots".into(),
version: 1,
challenge: Uuid::new_v4(),
bucket: bucket.to_owned(),
marker: marker.map(str::to_owned),
page_cap: MULTIPART_INDEX_PAGE_SIZE,
},
}
}
pub fn validate(&self) -> crate::disk::error::Result<()> {
let query = &self.multipart_index;
if !self.disk_id.is_empty()
|| self.metrics
|| !self.noop
|| query.operation != "list_multipart_index_roots"
|| query.version != 1
|| query.challenge.is_nil()
|| query.bucket.is_empty()
|| query.bucket.len() > 255
|| query.bucket.contains('/')
|| matches!(query.bucket.as_str(), "." | "..")
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
{
return Err(crate::disk::error::DiskError::MethodNotAllowed);
}
Ok(())
}
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
self.validate()?;
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
return Err(crate::disk::error::DiskError::FileCorrupt);
}
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
}
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
self.validate()?;
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
}
}
fn valid_root(root: &str) -> bool {
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
}
impl MultipartIndexPage {
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
if expected_disk.is_nil()
|| self.disk_id != expected_disk
|| self.epoch.is_nil()
|| self.bucket != bucket
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|| self
.roots
.iter()
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|| self
.next_marker
.as_ref()
.is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
{
return Err(crate::disk::error::DiskError::FileCorrupt);
}
Ok(())
}
}
impl MultipartIndexRpcReply {
/// This is structural validation only; the caller must also verify `proof`.
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
let query = &request.multipart_index;
request.validate()?;
if self.operation != query.operation
|| self.version != query.version
|| self.challenge != query.challenge
|| self.proof.len() != 32
{
return Err(crate::disk::error::DiskError::FileCorrupt);
}
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
}
}
/// A disk can attest coverage only after every page belongs to the same physical
/// identity and index generation. An interrupted/changed stream contributes no
/// partial candidates to the erasure set's coverage count.
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
where
F: FnMut(Option<String>) -> Fut,
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
{
let mut epoch = None;
let mut marker = None;
let mut roots = Vec::new();
loop {
let page = fetch(marker.clone()).await?;
page.validate(bucket, marker.as_deref(), identity)?;
if epoch.is_some_and(|epoch| epoch != page.epoch) {
return Err(crate::disk::error::DiskError::DiskNotFound);
}
epoch = Some(page.epoch);
roots.extend(page.roots);
marker = page.next_marker;
if marker.is_none() {
return Ok(roots);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
let request = MultipartIndexRpcRequest::new("bucket", None);
let id = Uuid::new_v4();
let mut reply = MultipartIndexRpcReply {
operation: request.multipart_index.operation.clone(),
version: 1,
challenge: request.multipart_index.challenge,
page: MultipartIndexPage {
bucket: "bucket".into(),
disk_id: id,
epoch: Uuid::new_v4(),
roots: vec!["a".repeat(64)],
next_marker: None,
},
proof: vec![0; 32],
};
reply.validate(&request, id).expect("valid structural envelope");
assert!(
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
"old challenge cannot replay"
);
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
reply.page.next_marker = Some("a".repeat(64));
assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
reply.page.next_marker = None;
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
}
#[test]
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
crate::runtime::sources::ensure_test_rpc_secret();
let mut request = MultipartIndexRpcRequest::new("bucket", None);
let mut page = MultipartIndexPage {
bucket: "bucket".into(),
disk_id: Uuid::new_v4(),
epoch: Uuid::new_v4(),
roots: vec![],
next_marker: None,
};
let canonical = request.canonical_response("disk", &page).expect("canonical response");
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
crate::cluster::rpc::verify_tonic_rpc_response_proof(
&request.canonical_response("disk", page).expect("canonical"),
&proof,
)
};
verify(&request, &page).expect("exact response");
request.multipart_index.marker = Some("0".repeat(64));
assert!(verify(&request, &page).is_err(), "cursor is signed");
request.multipart_index.marker = None;
page.epoch = Uuid::new_v4();
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
}
#[tokio::test]
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
let id = Uuid::new_v4();
let epoch = Uuid::new_v4();
for replace_disk in [false, true] {
let first = MultipartIndexPage {
bucket: "bucket".into(),
disk_id: id,
epoch,
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
};
let second = MultipartIndexPage {
bucket: "bucket".into(),
disk_id: if replace_disk { Uuid::new_v4() } else { id },
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
roots: vec![],
next_marker: None,
};
let mut replies = std::collections::VecDeque::from([first, second]);
let result = collect_pages("bucket", id, |marker| {
let page = replies.pop_front().expect("bounded fixture pages");
if marker.is_some() {
assert!(page.roots.is_empty());
}
std::future::ready(Ok(page))
})
.await;
assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
assert!(replies.is_empty(), "exercise the continuation response");
}
}
}
File diff suppressed because it is too large Load Diff
-10
View File
@@ -269,11 +269,6 @@ pub enum StorageError {
/// retained for operator inspection.
#[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
DeleteRecoveryLegacyAmbiguous,
/// Delete recovery exhausted the per-request work budget before it could
/// classify every residue. Retrying later makes progress, so callers treat
/// it like `SlowDown`; it stays distinct so the disk error round-trips.
#[error("delete recovery work limit reached; please retry")]
DeleteRecoveryWorkLimit,
#[error("Decommission not started")]
DecommissionNotStarted,
#[error("Decommission already running")]
@@ -485,7 +480,6 @@ impl From<DiskError> for StorageError {
DiskError::InvalidPath => StorageError::InvalidPath,
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
}
}
@@ -513,7 +507,6 @@ impl StorageError {
StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
StorageError::StorageFull => DiskError::DiskFull,
StorageError::SlowDown => DiskError::TooManyOpenFiles,
StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
StorageError::InconsistentDisk => DiskError::InconsistentDisk,
StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
@@ -717,7 +710,6 @@ impl Clone for StorageError {
StorageError::StorageFull => StorageError::StorageFull,
StorageError::SlowDown => StorageError::SlowDown,
StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
StorageError::InvalidUploadIDKeyCombination(a, b) => {
StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
@@ -860,7 +852,6 @@ impl StorageError {
StorageError::StorageFull => StorageErrorCode::StorageFull,
StorageError::SlowDown => StorageErrorCode::SlowDown,
StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
@@ -967,7 +958,6 @@ impl StorageError {
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
StorageErrorCode::PrefixAccessDenied => {
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
}
+4 -62
View File
@@ -79,14 +79,6 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
}
/// Validate that replicated compression metadata can be read by this build.
///
/// # Errors
/// Returns an error for an unsupported compression scheme.
pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
compression_scheme_to_read_plan(scheme).map(|_| ())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionBackend {
Legacy,
@@ -452,12 +444,10 @@ impl WritePlan {
if let Some(encryption) = self.encryption {
let etag_nonce = match encryption.mode {
WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
// Each part write needs an independent fingerprint, including a
// retry at the same part number. This salt affects only the ETag
// MAC; encryption framing and its data nonce stay unchanged.
WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
base_nonce
}
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
};
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
.with_inner_etag_delegation(delegate_inner_etag);
@@ -568,54 +558,6 @@ mod tests {
}
}
#[tokio::test]
async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
let plaintext = b"same confidential part";
let size = i64::try_from(plaintext.len()).expect("test length");
for object_key_mode in [false, true] {
let mut tags = std::collections::HashSet::new();
for part_number in [1, 2, 1] {
let encryption = if object_key_mode {
WriteEncryption::multipart_object_key([7; 32], part_number)
} else {
WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
};
let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
let mut reader = WritePlan::new()
.with_encryption(encryption)
.apply(source, size)
.expect("part plan");
let mut ciphertext = Vec::new();
reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
let tag = reader.try_resolve_etag().expect("part fingerprint");
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
assert!(
tags.insert(tag),
"different part numbers and retries must not identify matching plaintext"
);
#[cfg(feature = "rio-v2")]
let mut decrypt = if object_key_mode {
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
} else {
DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
};
#[cfg(not(feature = "rio-v2"))]
let mut decrypt = DecryptReader::new_multipart(
Cursor::new(ciphertext),
[7; 32],
if object_key_mode { [0; 12] } else { [9; 12] },
vec![part_number as usize],
);
let mut decoded = Vec::new();
decrypt
.read_to_end(&mut decoded)
.await
.expect("existing data nonce still decrypts");
assert_eq!(decoded, plaintext);
}
}
}
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
let plaintext = b"checksum-context-through-write-plan".repeat(256);
let actual_size = plaintext.len() as i64;
+38 -73
View File
@@ -28,6 +28,7 @@ use tokio_util::sync::CancellationToken;
#[derive(Clone)]
pub struct NamespaceLockFence {
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
local_probes: Arc<Vec<rustfs_lock::fast_lock::guard::FastLockGuardProbe>>,
#[cfg(test)]
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
}
@@ -44,6 +45,7 @@ impl NamespaceLockFence {
pub(crate) fn new() -> Self {
Self {
signals: Arc::default(),
local_probes: Arc::default(),
#[cfg(test)]
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
}
@@ -58,17 +60,28 @@ impl NamespaceLockFence {
{
return true;
}
self.signals.iter().any(|signal| signal.is_lost())
self.signals.iter().any(|signal| signal.is_lost()) || self.local_probes.iter().any(|probe| probe.is_lost())
}
pub(crate) fn add_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
Arc::make_mut(&mut self.signals).push(signal);
}
fn add_local_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
if let rustfs_lock::NamespaceLockGuard::Fast(guard) = guard
&& let Some(probe) = guard.release_probe()
{
Arc::make_mut(&mut self.local_probes).push(probe);
}
}
fn extend(&mut self, other: &Self) {
if !Arc::ptr_eq(&self.signals, &other.signals) {
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
}
if !Arc::ptr_eq(&self.local_probes, &other.local_probes) {
Arc::make_mut(&mut self.local_probes).extend(other.local_probes.iter().cloned());
}
#[cfg(test)]
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
@@ -106,6 +119,7 @@ impl NamespaceLockSignalTestFence {
) -> Self {
let fence = NamespaceLockFence {
signals: Arc::default(),
local_probes: Arc::default(),
forced_lost: Arc::new(vec![loss_handle]),
};
let signal_key = Arc::as_ptr(signal) as usize;
@@ -160,6 +174,20 @@ pub struct ObjectLockConfigSnapshot {
}
impl ObjectLockConfigSnapshot {
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
self._lifecycle_guard.is_some()
}
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
self.bucket.as_deref() == Some(bucket)
&& incarnation.is_some()
&& self.bucket_incarnation_id == incarnation
&& self
._lifecycle_guard
.as_ref()
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
}
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
Self {
store_id: None,
@@ -208,6 +236,7 @@ impl ObjectLockConfigSnapshot {
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
lifecycle_fence.add_signal(signal);
}
lifecycle_fence.add_local_guard(&lifecycle_guard);
Self {
store_id: Some(store_id),
bucket: Some(bucket.to_string()),
@@ -949,65 +978,8 @@ pub enum ShardIntegrityWriteMode {
Protected,
}
/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
/// Credits cover directory entries (including a minimum charge for an empty probe) and
/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
#[derive(Debug)]
pub struct DeleteRecoveryWorkBudget {
remaining: std::sync::atomic::AtomicU32,
}
/// Entries one disk may reserve up front per recovery step (discovery plus
/// inspection). Every disk of every set a request fans out to reserves this
/// much before any refund arrives, so a budget must cover that many disks.
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
impl Default for DeleteRecoveryWorkBudget {
/// Covers a full 16-drive set with headroom; `for_disks` scales it when a
/// request spans more drives than that.
fn default() -> Self {
Self::new(16 * 1024 * 1024)
}
}
impl DeleteRecoveryWorkBudget {
/// A per-request budget sized for `disks` concurrent up-front reservations.
pub(crate) fn for_disks(disks: usize) -> Self {
let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
let default = u64::from(Self::default().remaining());
Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
}
pub(crate) fn new(entries: u32) -> Self {
Self {
remaining: std::sync::atomic::AtomicU32::new(entries),
}
}
pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
use std::sync::atomic::Ordering;
self.remaining
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
(remaining > 0).then(|| remaining.saturating_sub(maximum))
})
.map(|remaining| remaining.min(maximum))
.map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
}
pub(crate) fn remaining(&self) -> u32 {
self.remaining.load(std::sync::atomic::Ordering::Acquire)
}
pub(crate) fn refund(&self, unused: u32) {
self.remaining.fetch_add(unused, std::sync::atomic::Ordering::Release);
}
}
#[derive(Default, Clone)]
pub struct ObjectOptions {
#[doc(hidden)]
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
// Use the maximum parity (N/2), used when saving server configuration files
pub max_parity: bool,
pub mod_time: Option<OffsetDateTime>,
@@ -1159,12 +1131,6 @@ pub struct ObjectOptions {
pub expiration: ExpirationOptions,
pub lifecycle_audit_event: LcAuditEvent,
/// In-process request gate evaluated on authoritative completion metadata.
/// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
/// implementations may own customer encryption keys.
#[doc(hidden)]
pub multipart_completion_authorization:
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
pub eval_metadata: Option<HashMap<String, String>>,
/// Internal compare-and-set condition for replication workers publishing
/// terminal status after remote I/O. Storage validates it while holding
@@ -1178,10 +1144,6 @@ pub struct ObjectOptions {
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
/// before a destructive commit reaches the set layer.
pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
/// ECStore single PUTs acquire their destructive-write configuration guards
/// only after staging the client body. External lock owners supply a snapshot.
#[doc(hidden)]
pub defer_put_object_lock_snapshot: bool,
pub want_checksum: Option<Checksum>,
pub skip_verify_bitrot: bool,
@@ -1213,11 +1175,6 @@ pub enum ReplicationStatusWritebackMode {
}
impl ObjectOptions {
pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
self.delete_recovery_work_budget
.get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
}
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
match self.shard_integrity_write_mode {
Some(ShardIntegrityWriteMode::Protected) => true,
@@ -1425,6 +1382,13 @@ impl ObjectOptions {
if let Some(signal) = guard.lock_lost_signal() {
self.add_namespace_lock_lost_signal(signal);
}
if let rustfs_lock::NamespaceLockGuard::Fast(local) = guard
&& !local.is_disabled()
{
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.add_local_guard(guard);
}
}
pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
@@ -1432,6 +1396,7 @@ impl ObjectOptions {
if let Some(signal) = guard.lock_lost_signal() {
fence.add_signal(signal);
}
fence.add_local_guard(guard);
}
pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
+39 -1
View File
@@ -58,7 +58,7 @@ use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -71,6 +71,9 @@ const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
// Bound abandoned speculative RPCs independently for each storage instance.
pub(crate) const SPECULATIVE_RESERVATION_LIMIT: usize = 64;
/// A lease is deliberately short-lived. The coordinator treats expiry as a
/// failed publication rather than silently continuing with a peer that may
/// have started movement after the lease was abandoned.
@@ -101,6 +104,12 @@ pub(crate) struct NamespaceCommitGuard {
counted: bool,
}
impl NamespaceCommitGuard {
pub(crate) fn covers_instance(&self, ctx: &Arc<InstanceContext>) -> bool {
Arc::ptr_eq(&self.ctx, ctx)
}
}
impl Drop for NamespaceCommitGuard {
fn drop(&mut self) {
if self.counted {
@@ -332,6 +341,7 @@ pub struct InstanceContext {
scanner_publication_state: AtomicU8,
namespace_commits: AtomicU64,
namespace_commit_generation: AtomicU64,
speculative_reservations: Arc<Semaphore>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
delete_recovery_hints: DeleteRecoveryHints,
@@ -392,6 +402,7 @@ impl InstanceContext {
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
namespace_commits: AtomicU64::new(0),
namespace_commit_generation: AtomicU64::new(0),
speculative_reservations: Arc::new(Semaphore::new(SPECULATIVE_RESERVATION_LIMIT)),
object_encryption_resolver: OnceLock::new(),
delete_recovery_hints: DeleteRecoveryHints::default(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -413,6 +424,11 @@ impl InstanceContext {
self.lock_manager.clone()
}
/// Skip speculative work when its detached owners already fill the instance budget.
pub(crate) fn try_acquire_speculative_reservation(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.speculative_reservations).try_acquire_owned().ok()
}
/// Run a multi-step mutation on its own task and wait for its output.
///
/// Dropping the returned future leaves the task running to completion, so
@@ -1143,6 +1159,26 @@ mod tests {
assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false));
}
#[test]
fn speculative_reservation_budget_is_bounded_and_instance_local() {
let first = InstanceContext::new();
let other = InstanceContext::new();
let mut permits = (0..super::SPECULATIVE_RESERVATION_LIMIT)
.map(|_| first.try_acquire_speculative_reservation().expect("initial permit"))
.collect::<Vec<_>>();
assert!(first.try_acquire_speculative_reservation().is_none());
assert!(
other.try_acquire_speculative_reservation().is_some(),
"budgets belong to their own instance"
);
drop(permits.pop());
let replacement = first
.try_acquire_speculative_reservation()
.expect("dropping an owner frees its slot");
assert!(first.try_acquire_speculative_reservation().is_none());
drop(replacement);
}
#[test]
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
let first = Arc::new(InstanceContext::new());
@@ -1151,6 +1187,8 @@ mod tests {
other.set_scanner_publication_state(false);
assert!(first.scanner_publication_state_allowed());
let one = first.begin_namespace_commit();
assert!(one.covers_instance(&first));
assert!(!one.covers_instance(&other));
let shared_owner = Arc::clone(&one);
let two = first.begin_namespace_commit();
assert!(first.namespace_commits_pending());
+3 -106
View File
@@ -318,21 +318,6 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
}
}
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
match rebalance_error_source(err) {
Error::Lock(err) => is_rebalance_transient_lock_error(err),
// No metadata was written when lock admission failed. Each attempt
// must acquire its quorum again before loading or saving this run.
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket,
object,
..
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
_ => false,
}
}
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
if err.kind() == std::io::ErrorKind::TimedOut {
return true;
@@ -400,7 +385,9 @@ where
match result {
Ok(value) => return Ok(value),
Err(err) => {
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
if attempt.saturating_add(1) >= max_attempts.max(1)
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
{
return Err(err);
}
#[cfg(test)]
@@ -732,82 +719,6 @@ mod error_source_tests {
}
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
let mut attempts = 0;
retry_rebalance_metadata_access(None, 3, || {
attempts += 1;
std::future::ready(if attempts == 1 {
resolve_rebalance_meta_save_result(
Err(rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)),
"stop_rebalance",
)
} else {
Ok(())
})
})
.await
.expect("a later quorum-protected admission must be allowed to succeed");
assert_eq!(attempts, 2);
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
for max_attempts in [0, 1, 3] {
let mut attempts = 0;
let result = retry_rebalance_metadata_access(None, max_attempts, || {
attempts += 1;
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)))
})
.await;
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
assert_eq!(attempts, max_attempts.max(1));
assert!(matches!(
rebalance_error_source(&error),
Error::NamespaceLockQuorumUnavailable {
mode: "write",
required: 1,
achieved: 0,
..
}
));
}
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
let mut attempts = 0;
let result = retry_rebalance_metadata_access(None, 3, || {
attempts += 1;
std::future::ready(Err::<(), _>(if attempts == 1 {
rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)
} else {
Error::OperationCanceled
}))
})
.await;
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
assert!(matches!(result, Err(Error::OperationCanceled)));
}
#[tokio::test]
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
for err in [
@@ -820,20 +731,6 @@ mod error_source_tests {
required: 3,
achieved: 2,
},
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: "data-bucket".to_string(),
object: REBAL_META_NAME.to_string(),
required: 1,
achieved: 0,
},
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
object: "other-metadata.bin".to_string(),
required: 1,
achieved: 0,
},
Error::other("stale rebalance run rejected: lock acquisition timed out"),
Error::other("rebalance distributed run fence lost"),
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
bytes.len()
};
Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+4 -3
View File
@@ -37,6 +37,7 @@ use std::{
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::{
select,
sync::{Notify, RwLock},
@@ -8182,7 +8183,7 @@ mod tests {
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
}
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
@@ -8247,7 +8248,7 @@ mod tests {
_opts: WarmBackendGetOpts,
) -> std::result::Result<ReadCloser, std::io::Error> {
if self.healthy {
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
} else {
Err(std::io::Error::other("mock get failed"))
}
@@ -9894,7 +9895,7 @@ mod tests {
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
}
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
@@ -2538,6 +2538,8 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
pub(in crate::set_disk) committed_file_info: FileInfo,
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
pub(in crate::set_disk) tail_has_owned_bucket_fences: bool,
}
pub(in crate::set_disk) struct RenameTailCleanup {
@@ -3468,6 +3470,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
rollback_receipt: Option<RenameRollbackReceipt>,
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
}
impl<'a> RenameDataFenceOptions<'a> {
@@ -3483,6 +3487,8 @@ impl<'a> RenameDataFenceOptions<'a> {
rollback_receipt: None,
multipart_proof_authority: None,
multipart_commit_owner: None,
reservation_ctx: None,
prepared_reservations: None,
}
}
@@ -3519,6 +3525,415 @@ impl<'a> RenameDataFenceOptions<'a> {
self.namespace_commit_guard = namespace_commit_guard;
self
}
pub(in crate::set_disk) fn with_reservation_context(
mut self,
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
) -> Self {
self.reservation_ctx = ctx;
self.prepared_reservations = reservations;
self
}
}
#[derive(Clone)]
struct MetadataRenewalTarget {
disk: DiskStore,
token: Uuid,
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
}
struct MetadataRenewalGroup {
targets: Arc<parking_lot::Mutex<HashMap<usize, MetadataRenewalTarget>>>,
task: tokio::task::JoinHandle<()>,
}
impl Drop for MetadataRenewalGroup {
fn drop(&mut self) {
self.task.abort();
}
}
pub(crate) struct RenameMetadataReservation {
disk: DiskStore,
token: Option<Uuid>,
// Remote reservation protocol v2 and the local owned path both retain
// bucket publication and lifecycle guards until the physical write drains.
owns_bucket_fence: bool,
short: bool,
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
renewal: Option<(Arc<MetadataRenewalGroup>, usize)>,
}
impl RenameMetadataReservation {
fn stop_renewal(&mut self) {
if let Some((group, index)) = self.renewal.take() {
let mut targets = group.targets.lock();
targets.remove(&index);
if targets.is_empty() {
group.task.abort();
}
}
}
async fn cancel(&mut self) -> disk::error::Result<()> {
self.stop_renewal();
let Some(token) = self.token else { return Ok(()) };
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
Ok(Ok(true)) => {
self.token = None;
Ok(())
}
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
Ok(Err(error)) => Err(error),
Err(_) => Err(DiskError::Timeout),
}
}
}
impl Drop for RenameMetadataReservation {
fn drop(&mut self) {
self.stop_renewal();
let Some(token) = self.token.take() else { return };
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
let disk = self.disk.clone();
runtime.spawn(async move {
let _ = disk.cancel_object_metadata_reservation(token).await;
});
}
}
}
#[derive(Clone, Copy)]
pub(crate) enum RenameReservationMode {
Ownerless,
Owned,
ShortOwned,
}
pub(crate) async fn prepare_rename_metadata_reservations(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
incarnation: Uuid,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
owns_bucket_fence: bool,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
prepare_rename_metadata_reservations_until_abandoned(
disks,
bucket,
object,
incarnation,
ctx,
if owns_bucket_fence {
RenameReservationMode::Owned
} else {
RenameReservationMode::Ownerless
},
std::future::pending(),
)
.await
}
pub(crate) async fn prepare_rename_metadata_reservations_until_abandoned(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
incarnation: Uuid,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
mode: RenameReservationMode,
abandoned: impl std::future::Future<Output = ()>,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
let owns_bucket_fence = !matches!(mode, RenameReservationMode::Ownerless);
let short = matches!(mode, RenameReservationMode::ShortOwned);
let mut abandoned = Box::pin(abandoned);
let discarded = Arc::new(std::sync::atomic::AtomicBool::new(false));
let mut pending: FuturesUnordered<_> = disks
.iter()
.enumerate()
.map(|(_index, disk)| {
let discarded = Arc::clone(&discarded);
async move {
let result: disk::error::Result<Option<RenameMetadataReservation>> = async {
if discarded.load(std::sync::atomic::Ordering::Acquire) {
return Ok(None);
}
let Some(disk) = disk else { return Ok(None) };
#[cfg(test)]
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_BEFORE).await;
if disk.is_local() {
crate::store::admit_local_disk(ctx, disk, None, false).await?;
}
if discarded.load(std::sync::atomic::Ordering::Acquire) {
return Ok(None);
}
let prepared_at = tokio::time::Instant::now();
let token = if disk.is_local() && owns_bucket_fence {
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
.await
.ok_or(DiskError::DiskOngoingReq)?;
if short {
store
.reserve_local_object_metadata_short(
&disk.endpoint().to_string(),
bucket,
object,
incarnation,
Duration::from_millis(50),
)
.await?
} else {
store
.reserve_local_object_metadata_at_incarnation(
&disk.endpoint().to_string(),
bucket,
object,
incarnation,
Duration::from_secs(5),
)
.await?
}
} else if short {
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
return Err(DiskError::InvalidPath);
};
remote
.reserve_object_metadata_short(bucket, object, incarnation, Duration::from_millis(50))
.await?
} else {
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
.await?
};
// No await separates receipt from installing the cancellation owner.
let reservation = RenameMetadataReservation {
disk: disk.clone(),
token: Some(token),
owns_bucket_fence: !disk.is_local() || owns_bucket_fence,
short,
renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)),
renewal: None,
};
if short {
// Install the cancellation owner before extending the short token.
// A dropped renew response still leaves us the token to cancel.
disk.renew_object_metadata(bucket, object, token, Duration::from_millis(50))
.await?;
}
#[cfg(test)]
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_AFTER).await;
Ok(Some(reservation))
}
.await;
(_index, result)
}
})
.collect();
let mut reservations: Vec<Option<RenameMetadataReservation>> = (0..disks.len()).map(|_| None).collect();
let mut discard = false;
while !pending.is_empty() {
tokio::select! {
biased;
_ = &mut abandoned, if !discard => {
discard = true;
discarded.store(true, std::sync::atomic::Ordering::Release);
reservations.clear();
}
result = pending.next() => {
let Some((index, result)) = result else { break };
if discard {
drop(result);
} else {
match result {
Ok(reservation) => reservations[index] = reservation,
Err(_) => {
discard = true;
discarded.store(true, std::sync::atomic::Ordering::Release);
reservations.clear();
}
}
}
}
}
}
if discard {
// Drain all in-flight preparations so late tokens get a cancellation owner.
return Ok(None);
}
let targets = Arc::new(parking_lot::Mutex::new(HashMap::new()));
for (index, reservation) in reservations.iter().enumerate() {
if let Some(reservation) = reservation {
let token = reservation.token.ok_or(DiskError::Unexpected)?;
targets.lock().insert(
index,
MetadataRenewalTarget {
disk: reservation.disk.clone(),
token,
renewed_at: Arc::clone(&reservation.renewed_at),
},
);
}
}
if !targets.lock().is_empty() {
let renewal_targets = Arc::clone(&targets);
let renewal_ctx = Arc::clone(ctx);
let renewal_bucket = bucket.to_owned();
let renewal_object = object.to_owned();
// Short leases remain short even after renewal, so a lost renewal and
// cancellation cannot strand a bucket fence for the legacy lease time.
let task = tokio::spawn(async move {
let interval = if short {
Duration::from_millis(500)
} else {
Duration::from_secs(20)
};
let retry = if short {
Duration::from_millis(250)
} else {
Duration::from_secs(5)
};
let lifetime = if short {
Duration::from_secs(2)
} else {
Duration::from_secs(60)
};
let mut delay = interval;
loop {
tokio::time::sleep(delay).await;
let current: Vec<_> = renewal_targets.lock().values().cloned().collect();
if current.is_empty() {
break;
}
// Once any acknowledgment is too old, the complete fanout can
// no longer be safely activated. Stop retrying stale peers;
// their receiver-side tokens still expire on their own.
if current.iter().any(|target| target.renewed_at.lock().elapsed() >= lifetime) {
break;
}
let results = join_all(current.into_iter().map(|target| {
let ctx = Arc::clone(&renewal_ctx);
let bucket = &renewal_bucket;
let object = &renewal_object;
async move {
let started = tokio::time::Instant::now();
let admitted = !target.disk.is_local()
|| crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok();
if admitted
&& target
.disk
.renew_object_metadata(bucket, object, target.token, retry)
.await
.is_ok()
{
*target.renewed_at.lock() = started;
true
} else {
false
}
}
}))
.await;
delay = if results.into_iter().all(|success| success) {
interval
} else {
retry
};
}
});
let group = Arc::new(MetadataRenewalGroup { targets, task });
for (index, reservation) in reservations.iter_mut().enumerate() {
if let Some(reservation) = reservation {
reservation.renewal = Some((Arc::clone(&group), index));
}
}
}
Ok(Some(reservations))
}
pub(crate) fn align_rename_metadata_reservations(
mut reservations: Vec<Option<RenameMetadataReservation>>,
disks: &[Option<DiskStore>],
) -> Option<Vec<Option<RenameMetadataReservation>>> {
if reservations.len() != disks.len() {
return None;
}
let mut aligned = Vec::with_capacity(disks.len());
for disk in disks {
let Some(disk) = disk else {
aligned.push(None);
continue;
};
let index = reservations
.iter()
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
aligned.push(reservations[index].take());
}
// Disks lost while staging need no activation; dropping their still-inert
// reservations schedules cancellation without blocking the commit.
Some(aligned)
}
async fn activate_rename_metadata_reservations(
mut reservations: Vec<Option<RenameMetadataReservation>>,
bucket: &str,
object: &str,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
// Short tokens continue renewing during activation; an activated token
// can otherwise expire while another disk's activation is still pending.
// Long tokens retain their existing non-renewable queued behavior.
for reservation in reservations.iter_mut().flatten() {
if !reservation.short {
reservation.stop_renewal();
}
}
if reservations.iter().flatten().any(|reservation| {
let max_age = if reservation.short {
Duration::from_secs(1)
} else {
Duration::from_secs(50)
};
reservation.renewed_at.lock().elapsed() >= max_age
}) {
return Ok(None);
}
#[cfg(test)]
let test_reservations = &reservations;
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
if reservation.disk.is_local() {
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
}
let token = reservation.token.ok_or(DiskError::Unexpected)?;
let result = reservation
.disk
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
.await;
#[cfg(test)]
if result.is_ok() {
let index = test_reservations
.iter()
.position(|candidate| {
candidate
.as_ref()
.is_some_and(|candidate| std::ptr::eq(candidate, reservation))
})
.expect("activated reservation belongs to the fanout");
rename_fanout_barrier::checkpoint(object, index, rename_fanout_barrier::PHASE_RESERVATION_ACTIVATED).await;
}
result
}))
.await;
for reservation in reservations.iter_mut().flatten() {
reservation.stop_renewal();
}
if activated.into_iter().any(|result| result.is_err()) {
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
if cancelled.into_iter().any(|result| result.is_err()) {
return Err(DiskError::DiskOngoingReq);
}
return Ok(None);
}
Ok(Some(reservations))
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3530,7 +3945,7 @@ type RenameDataLegacyTuple = (
Option<OldCurrentSize>,
);
fn put_rename_early_ack_enabled() -> bool {
pub(crate) fn put_rename_early_ack_enabled() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
}
@@ -3896,6 +4311,8 @@ impl SetDisks {
old_current_source,
committed_file_info,
tail_drain: None,
tail_has_metadata_reservations: false,
tail_has_owned_bucket_fences: false,
})
}
@@ -3992,6 +4409,8 @@ impl SetDisks {
rollback_receipt,
multipart_proof_authority: _,
multipart_commit_owner: _,
reservation_ctx,
prepared_reservations,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4007,6 +4426,21 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
(_, _) => None,
};
let tail_has_metadata_reservations = reservations.is_some();
let tail_has_owned_bucket_fences = reservations.as_ref().is_some_and(|reservations| {
reservations.len() == disks.len()
&& disks.iter().any(Option::is_some)
&& disks.iter().zip(reservations).all(|(disk, reservation)| match disk {
Some(disk) => reservation
.as_ref()
.is_some_and(|owner| owner.owns_bucket_fence && Arc::ptr_eq(&owner.disk, disk)),
None => true,
})
});
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
@@ -4039,6 +4473,11 @@ impl SetDisks {
let file_info = file_info.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let namespace_commit_guard = namespace_commit_guard.clone();
let reservation = reservations
.as_mut()
.and_then(|entries| entries.get_mut(i))
.and_then(Option::take);
let reservation_ctx = reservation_ctx.clone();
tasks.spawn(async move {
let _namespace_commit_guard = namespace_commit_guard.clone();
let mut dispatch_state = RenameDispatchState::NotDispatched;
@@ -4049,6 +4488,17 @@ impl SetDisks {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let disk_admission = if reservation.is_some() && disk.is_local() {
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
// The fanout owner already covers this instance through
// physical completion; only add a disk owner when it does not.
let covered = namespace_commit_guard
.as_ref()
.is_some_and(|guard| guard.covers_instance(ctx));
crate::store::admit_local_disk(ctx, &disk, None, !covered).await?
} else {
None
};
let disk_wait_location_stage = if disk.is_local() {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
} else {
@@ -4074,6 +4524,11 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let _foreground_active_guard = rename_foreground_active_guard(i);
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
} else {
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
};
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
@@ -4083,9 +4538,8 @@ impl SetDisks {
&dst_object,
disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
namespace_owner,
..Default::default()
},
)
@@ -4363,6 +4817,8 @@ impl SetDisks {
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
commit.map(|mut commit| {
commit.tail_drain = Some(tail_drain);
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
commit.tail_has_owned_bucket_fences = tail_has_owned_bucket_fences;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
attach_tail_owner_started,
@@ -4432,6 +4888,8 @@ impl SetDisks {
rollback_receipt,
multipart_proof_authority,
multipart_commit_owner,
reservation_ctx: _,
prepared_reservations: _,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4901,6 +5359,8 @@ impl SetDisks {
old_current_source,
committed_file_info,
tail_drain: None,
tail_has_metadata_reservations: false,
tail_has_owned_bucket_fences: false,
})
}
@@ -7358,6 +7818,9 @@ pub(crate) mod rename_fanout_barrier {
MetadataSlowtailFault,
NonInlineHedgeTimer,
LateMetadataRefresh,
ReservationBefore,
ReservationAfter,
ReservationActivated,
}
impl From<&'static str> for Phase {
@@ -7380,6 +7843,9 @@ pub(crate) mod rename_fanout_barrier {
pub const PHASE_ROLLBACK: Phase = Phase::Rollback;
pub const PHASE_READ_VERSION: Phase = Phase::ReadVersion;
pub const PHASE_METADATA_SLOWTAIL_FAULT: Phase = Phase::MetadataSlowtailFault;
pub const PHASE_RESERVATION_BEFORE: Phase = Phase::ReservationBefore;
pub const PHASE_RESERVATION_AFTER: Phase = Phase::ReservationAfter;
pub const PHASE_RESERVATION_ACTIVATED: Phase = Phase::ReservationActivated;
/// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk.
pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer;
@@ -7454,6 +7920,18 @@ pub(crate) mod rename_fanout_barrier {
}
impl BarrierHandle {
/// Keep the current task parked without intercepting later tasks.
pub fn disarm(&self) {
let mut reg = lock();
if reg
.armed
.get(&self.key)
.is_some_and(|armed| Arc::ptr_eq(&armed.release, &self.release))
{
reg.armed.remove(&self.key);
}
}
/// Await until the armed fan-out task has parked at the checkpoint. Uses a
/// stored-permit `Notify`, so this is race-free regardless of whether the
/// task reaches the checkpoint before or after this call — no sleeps.
@@ -7619,6 +8097,45 @@ mod tests {
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() {
let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await;
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let endpoint = disk.endpoint().to_string();
ctx.local_disk_map()
.write()
.await
.insert(endpoint.clone(), Some(disk.clone()));
let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false)
.await
.expect("prepare")
.expect("complete fanout");
let group = Arc::clone(
&reservations[0]
.as_ref()
.expect("reservation")
.renewal
.as_ref()
.expect("renewal")
.0,
);
ctx.local_disk_map().write().await.remove(&endpoint);
tokio::task::yield_now().await;
tokio::time::pause();
tokio::time::advance(Duration::from_secs(21)).await;
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(41)).await;
for _ in 0..10 {
if group.task.is_finished() {
break;
}
tokio::task::yield_now().await;
}
tokio::time::resume();
assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts");
drop(reservations);
}
#[tokio::test]
async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() {
use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint};
+1 -8
View File
@@ -195,14 +195,7 @@ impl SetDisks {
metadata.add_version(selected)?;
}
if metadata.versions.is_empty() {
// With no decodable version, the per-version quorum loop cannot
// reject corrupt or unavailable replicas. Only complete, readable
// evidence may establish that this key has no authoritative version.
return if complete {
Ok(None)
} else {
Err(DiskError::ErasureReadQuorum)
};
return Ok(None);
}
Ok(Some(MetaCacheEntry {
name,
+18 -32
View File
@@ -357,7 +357,8 @@ use crate::io_support::rio::HashReader;
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
pub const MAX_PARTS_COUNT: usize = 10000;
pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
@@ -409,6 +410,10 @@ impl ObjectLockDiagGuard {
self
}
async fn downgrade_committed_tail(&mut self) -> bool {
self.guard.downgrade_committed_tail().await
}
/// Whether the underlying namespace lock's heartbeat has observed a
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
/// point on this so a stale lock holder does not race a double-write.
@@ -901,6 +906,10 @@ mod core;
pub(crate) use core::io_primitives::disk_call_counters;
#[cfg(test)]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
pub(crate) use core::io_primitives::{
RenameMetadataReservation, RenameReservationMode, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
prepare_rename_metadata_reservations_until_abandoned, put_rename_early_ack_enabled,
};
mod ctx;
mod metadata;
mod ops;
@@ -927,6 +936,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(all(test, feature = "test-util"))]
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
#[cfg(test)]
pub(crate) use ops::object::pause_put_object_commit;
#[cfg(test)]
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
#[cfg(any(test, feature = "test-util"))]
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
@@ -4695,17 +4706,6 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
}
// Reusing a held writer additionally requires the same quorum, so repeated
// clients must not give different weight to an otherwise identical domain.
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
!left.is_empty()
&& left.len() == right.len()
&& left.iter().all(|client| {
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
})
}
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5655,9 +5655,9 @@ impl SetDisks {
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
// Distributed lock clients receive the raw ObjectKey; the pool
// namespace is not part of the key stored by their backends.
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
(true, true) => {
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
}
_ => false,
}
}
@@ -9892,7 +9892,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
async fn namespace_lock_domain_includes_pool_namespace() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9931,25 +9931,11 @@ mod tests {
"sets in the same pool share the object namespace lock domain"
);
assert!(
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
"different pools using identical backends lock the same raw object key"
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
"different pool namespaces must not be deduplicated solely by identical clients"
);
}
#[test]
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
assert!(!same_distributed_lock_quorum(
&[a.clone(), a.clone(), b.clone()],
&[a.clone(), b.clone(), b]
));
assert!(!same_distributed_lock_quorum(&[], &[]));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn streaming_reader_holds_read_lock_until_eof() {
@@ -141,7 +141,6 @@ pub(crate) struct DeleteRecoverySummary {
#[derive(Clone)]
pub(crate) struct DeleteRecoveryContext<'a> {
pub incarnation: Option<Uuid>,
pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
pub dry_run: bool,
pub explicit_delete: bool,
/// Only the locked heal path may repair a live marker-creation source.
@@ -157,7 +156,6 @@ impl<'a> DeleteRecoveryContext<'a> {
pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
Self {
incarnation,
work_budget: None,
dry_run: false,
explicit_delete: false,
repair_marker_source: false,
@@ -232,54 +230,19 @@ async fn inspect(
bucket: &str,
object: &str,
transaction: Uuid,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> Vec<Result<DeleteRecoveryInspection>> {
join_all(disks.iter().map(|disk| async move {
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
let state = if let Some(budget) = budget {
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
let reply = disk
.inspect_delete_transaction_bounded(bucket, object, transaction, reserved)
.await?;
if reply.visited > reserved {
return Err(DiskError::FileCorrupt);
}
budget.refund(reserved - reply.visited.max(1));
reply.inspection
} else {
disk.inspect_delete_transaction(bucket, object, transaction).await?
};
let state = disk
.as_ref()
.ok_or(DiskError::DiskNotFound)?
.inspect_delete_transaction(bucket, object, transaction)
.await?;
validate(&state, object, transaction)?;
Ok(state)
}))
.await
}
async fn discover_bounded(
disk: &DiskStore,
bucket: &str,
object: &str,
budget: &crate::object_api::DeleteRecoveryWorkBudget,
) -> Result<Vec<DeleteTransactionId>> {
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
let page = disk.discover_delete_transactions_bounded(bucket, object, reserved).await?;
if page.visited > reserved
|| page.transactions.len() > 256
|| page
.transactions
.iter()
.any(|id| id.object != object || id.transaction.is_nil())
|| (!page.complete && !page.transactions.is_empty())
{
return Err(DiskError::FileCorrupt);
}
budget.refund(reserved - page.visited.max(1));
if !page.complete {
return Err(DiskError::DeleteRecoveryWorkLimit);
}
Ok(page.transactions)
}
async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
let mut cursor = None;
let mut ids = BTreeSet::new();
@@ -310,92 +273,30 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
.collect())
}
#[derive(Clone, Copy)]
struct RecoveryWorkScope<'a> {
namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
}
async fn settle_outcome(
disk: &DiskStore,
bucket: &str,
state: &DeleteRecoveryInspection,
action: DeleteRecoveryAction,
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
scope: RecoveryWorkScope<'_>,
) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
use crate::disk::DeleteRecoverySettlementOutcome;
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let outcome = if let Some(budget) = scope.budget {
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
let reply = disk
.settle_delete_transaction_bounded(
bucket,
crate::disk::BoundedDeleteRecoverySettlement {
object: state.id.object.clone(),
transaction: state.id.transaction,
expected_fingerprint: state.fingerprint,
action,
max_entries: reserved,
},
scope.namespace_owner.cloned(),
)
.await?;
if reply.visited > reserved
|| (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
{
return Err(DiskError::FileCorrupt);
}
budget.refund(reserved - reply.visited.max(1));
reply.outcome
} else {
DeleteRecoverySettlementOutcome::Inspected(Box::new(
disk.settle_delete_transaction_with_namespace_owner(
bucket,
&state.id.object,
state.id.transaction,
state.fingerprint,
action,
scope.namespace_owner.cloned(),
)
.await?,
))
};
if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
validate(next, &state.id.object, state.id.transaction)?;
}
Ok(outcome)
}
async fn settle(
disk: &DiskStore,
bucket: &str,
state: &DeleteRecoveryInspection,
action: DeleteRecoveryAction,
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
scope: RecoveryWorkScope<'_>,
namespace_owner: Option<&Arc<dyn Send + Sync>>,
) -> Result<DeleteRecoveryInspection> {
match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
Err(DiskError::DeleteRecoveryWorkLimit)
}
}
}
fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
}
fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
match outcome {
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
Err(_) => false,
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let result = disk
.settle_delete_transaction_with_namespace_owner(
bucket,
&state.id.object,
state.id.transaction,
state.fingerprint,
action,
namespace_owner.cloned(),
)
.await;
let next = result?;
validate(&next, &state.id.object, state.id.transaction)?;
Ok(next)
}
/// Each phase fans out across independent drives, and awaits all started work
@@ -406,16 +307,23 @@ async fn settle_group(
copies: &mut [DeleteRecoveryInspection],
action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
scope: RecoveryWorkScope<'_>,
namespace_owner: Option<&Arc<dyn Send + Sync>>,
removed_versions: &mut HashSet<Uuid>,
) -> Result<bool> {
let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
let action = action(copy);
async move {
match action {
Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
.await
.map(Some),
Some(action) => settle(
disk.as_ref().ok_or(DiskError::DiskNotFound)?,
bucket,
copy,
action,
can_commit,
namespace_owner,
)
.await
.map(Some),
None => Ok(None),
}
}
@@ -531,31 +439,16 @@ impl SetDisks {
true
}
};
let mut fast_receipt_quorum = false;
if can_try_local_fast_path && can_commit() {
let fast_published = join_all(disks.iter().map(|disk| async {
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
disk.try_publish_prepared_delete_local(
bucket,
object,
transaction,
context.namespace_owner.clone(),
context.work_budget.clone(),
)
.await
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
.await
}))
.await;
fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
if fast_receipt_quorum
&& fast_published
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return Ok(());
}
let all_fast_published = fast_published.len() == self.set_drive_count
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
if all_fast_published && can_commit() {
@@ -563,7 +456,6 @@ impl SetDisks {
test_hooks::record_cleanup_attempt(bucket, object);
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
let ns_owner = context.namespace_owner.clone();
let work_budget = context.work_budget.clone();
async move {
if !can_commit() {
return Ok(false);
@@ -574,24 +466,11 @@ impl SetDisks {
let Ok(Some((journal, _, _))) = pub_res else {
return Ok(false);
};
disk.try_finish_and_finalize_prepared_delete_local(
bucket,
object,
transaction,
journal,
ns_owner,
work_budget,
)
.await
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
.await
}
}))
.await;
if finalized
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return Ok(());
}
if finalized.iter().all(|res| matches!(res, Ok(true))) {
if let Some(ticket) = context.recovery_ticket.as_ref() {
self.ctx.delete_recovery_hints().ack(ticket);
@@ -600,17 +479,7 @@ impl SetDisks {
}
}
}
let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
if states
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return if fast_receipt_quorum {
Ok(())
} else {
Err(DiskError::DeleteRecoveryWorkLimit)
};
}
let states = inspect(disks, bucket, object, transaction).await;
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
&& states.iter().all(|state| {
@@ -630,50 +499,25 @@ impl SetDisks {
if state.state == DeleteRecoveryState::LegacyAmbiguous {
return Err(DiskError::FileCorrupt);
}
settle_outcome(
settle(
disk,
bucket,
state,
DeleteRecoveryAction::PublishCommit,
can_commit,
RecoveryWorkScope {
namespace_owner: context.namespace_owner.as_ref(),
budget: context.work_budget.as_deref(),
},
context.namespace_owner.as_ref(),
)
.await
}))
.await;
if !has_receipt_quorum(&published, write_quorum) {
return Err(
if published
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
DiskError::DeleteRecoveryWorkLimit
} else {
DiskError::ErasureWriteQuorum
},
);
if published
.iter()
.filter(|state| matches!(state, Ok(state) if receipt(state.state)))
.count()
< write_quorum
{
return Err(DiskError::ErasureWriteQuorum);
}
if published.iter().any(|outcome| {
matches!(
outcome,
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
)
}) {
return Ok(());
}
let published = published
.into_iter()
.map(|outcome| match outcome {
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
Err(DiskError::DeleteRecoveryWorkLimit)
}
Err(error) => Err(error),
})
.collect::<Vec<_>>();
// Physical cleanup is retryable after the certified decision. An
// unavailable participant retains the decision for its eventual return.
let ticket = context.recovery_ticket.clone();
@@ -826,7 +670,6 @@ impl SetDisks {
}
let DeleteRecoveryContext {
incarnation,
work_budget,
dry_run,
explicit_delete,
repair_marker_source,
@@ -838,13 +681,7 @@ impl SetDisks {
} = context;
let disks = self.disk_inventory().await;
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
let pages = join_all(disks.iter().flatten().map(|disk| async {
match work_budget.as_deref() {
Some(budget) => discover_bounded(disk, bucket, object, budget).await,
None => discover(disk, bucket, Some(object)).await,
}
}))
.await;
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
let mut transactions = BTreeSet::new();
let mut inspected = 0;
let mut offline = !complete_inventory;
@@ -875,7 +712,6 @@ impl SetDisks {
..Default::default()
};
let mut all_states = BTreeMap::new();
let mut registered_evidence_hint = false;
for transaction in transactions {
let mut complete = Vec::with_capacity(disks.len());
let inspections = if let Some((published_tx, published_states)) = published_hint
@@ -887,14 +723,8 @@ impl SetDisks {
{
published_states.to_vec()
} else {
inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
inspect(&disks, bucket, object, transaction).await
};
if inspections
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return Err(DiskError::DeleteRecoveryWorkLimit);
}
for state in inspections {
match state {
Ok(state) => {
@@ -906,33 +736,6 @@ impl SetDisks {
summary.blocked += 1;
return Ok(summary);
}
// Candidate UUID directories are not evidence. Only a bound, inspected
// journal can create a foreground hint; empty/partial scans rely on the
// existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
if work_budget.is_some()
&& !registered_evidence_hint
&& journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
&& journal.volume == bucket
&& journal.object == object
&& journal.transaction == transaction
&& matches!(
state.state,
DeleteRecoveryState::NewPrepared
| DeleteRecoveryState::QuorumCommitted
| DeleteRecoveryState::CommitFinished
)
{
self.ctx.delete_recovery_hints().register(
crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: self.pool_index,
set: self.set_index,
bucket: bucket.to_owned(),
object: object.to_owned(),
incarnation,
},
);
registered_evidence_hint = true;
}
}
complete.push(state);
}
@@ -964,7 +767,7 @@ impl SetDisks {
};
if let Some((transaction, source)) = prerequisite {
Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
let refreshed = inspect(&disks, bucket, object, transaction)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
@@ -1050,7 +853,7 @@ impl SetDisks {
if mutated {
// A sibling may have restored metadata or reclaimed shared
// evidence. Refresh fingerprints before acting on this group.
*copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
*copies = inspect(&disks, bucket, object, transaction)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
@@ -1073,10 +876,7 @@ impl SetDisks {
(copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
},
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?
@@ -1187,10 +987,7 @@ impl SetDisks {
)
},
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?;
@@ -1204,10 +1001,7 @@ impl SetDisks {
.then_some(DeleteRecoveryAction::FinishCommit)
},
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?;
@@ -1254,10 +1048,7 @@ impl SetDisks {
copies,
|copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?
@@ -1285,7 +1076,7 @@ impl SetDisks {
settled.push(copies);
} else {
for transaction in all_states.keys() {
let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
let fresh = inspect(&disks, bucket, object, *transaction).await;
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
summary.pending += 1;
return Ok(summary);
@@ -1295,7 +1086,7 @@ impl SetDisks {
}
for (index, mut copies) in settled.into_iter().enumerate() {
if index > 0 {
copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
copies = inspect(&disks, bucket, object, copies[0].id.transaction)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
@@ -1315,10 +1106,7 @@ impl SetDisks {
&mut copies,
|copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?
@@ -1570,457 +1358,6 @@ mod tests {
body
}
#[tokio::test]
async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
let bucket = "recover-bounded-mixed-quorum";
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &vec![73; 4096], false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let mut outcomes = Vec::new();
for (index, disk) in disks.iter().enumerate() {
let state = disk
.inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
.await
.expect("preparation inspection");
let max_entries = match index {
0 => 65536,
1 | 2 => state.visited,
_ => 1,
};
outcomes.push(
disk.settle_delete_transaction_bounded(
bucket,
crate::disk::BoundedDeleteRecoverySettlement {
object: "object".to_owned(),
transaction,
expected_fingerprint: state.inspection.fingerprint,
action: DeleteRecoveryAction::PublishCommit,
max_entries,
},
None,
)
.await
.map(|reply| reply.outcome),
);
}
assert!(
matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
);
assert!(matches!(
&outcomes[1],
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
));
assert!(outcomes[3].is_err());
assert!(has_receipt_quorum(&outcomes, 3));
assert!(!has_receipt_quorum(&outcomes, 4));
outcomes[2] = Err(DiskError::FileAccessDenied);
assert!(
!has_receipt_quorum(&outcomes, 3),
"a write failure cannot substitute for a durable receipt"
);
}
#[tokio::test]
async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
use crate::object_api::DeleteRecoveryWorkBudget;
use crate::storage_api_contracts::object::ObjectToDelete;
let bucket = "recover-bounded-request";
let (dirs, store, _) = fixture(bucket).await;
for dir in &dirs {
let prefix = dir.path().join(bucket).join("prefix");
std::fs::create_dir_all(&prefix).expect("physical prefix");
for index in 0..64 {
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
}
}
let error = store
.delete_object(
bucket,
"prefix",
ObjectOptions {
delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
..Default::default()
},
)
.await
.expect_err("incomplete discovery is not absence");
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
for incarnation in [
None,
Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation"),
),
] {
assert!(
!store
.ctx
.delete_recovery_hints()
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "prefix".to_owned(),
incarnation,
}),
"an unproven prefix must not move its scan into background recovery"
);
}
for distinct_versions in [false, true] {
let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
let objects = (0..1000)
.map(|_| ObjectToDelete {
object_name: "prefix".to_owned(),
version_id: distinct_versions.then(Uuid::new_v4),
..Default::default()
})
.collect();
let (_, errors) = store
.delete_objects(
bucket,
objects,
ObjectOptions {
delete_recovery_work_budget: Some(Arc::clone(&budget)),
..Default::default()
},
)
.await;
assert!(
errors
.iter()
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
"repeated actions must not each obtain a new allowance: {distinct_versions}"
);
assert!(
dirs.iter()
.all(|dir| dir.path().join(bucket).join("prefix/child-63").is_dir())
);
}
let error = store
.delete_object(bucket, "prefix", ObjectOptions::default())
.await
.expect_err("ordinary missing key");
assert!(
crate::error::is_err_object_not_found(&error),
"small complete inventory retains no-op semantics: {error:?}"
);
}
#[tokio::test]
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
use crate::layout::endpoint::Endpoint;
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::runtime::instance::InstanceContext;
let mut dirs = Vec::new();
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for set_index in 0..2 {
for disk_index in 0..4 {
let dir = tempfile::tempdir().expect("temporary disk");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(set_index);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
dirs.push(dir);
}
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 2,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("bounded-recovery-{pool_index}"),
platform: "test".to_owned(),
});
}
let endpoints = EndpointServerPools(pools);
let ctx = Arc::new(InstanceContext::new());
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
.await
.expect("local disks");
let cancel = tokio_util::sync::CancellationToken::new();
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
.await
.expect("two pools with two sets");
cancel.cancel();
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "recover-bounded-fanout";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
let mut keys = BTreeSet::new();
for pool in &store.pools {
for set in &pool.disk_set {
keys.insert(
candidates
.iter()
.find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
.expect("fixture key for set")
.clone(),
);
}
}
let keys = keys.into_iter().collect::<Vec<_>>();
for pool in &store.pools {
for set in &pool.disk_set {
assert!(
keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
"fixture must cover every set"
);
}
}
for dir in &dirs {
for key in &keys {
let prefix = dir.path().join(bucket).join(key);
std::fs::create_dir_all(&prefix).expect("physical prefix");
for child in 0..8 {
std::fs::create_dir(prefix.join(format!("child-{child}"))).expect("child");
}
}
}
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
let objects = (0..1000)
.map(|index| crate::storage_api_contracts::object::ObjectToDelete {
object_name: keys[index % keys.len()].clone(),
version_id: Some(Uuid::new_v4()),
..Default::default()
})
.collect();
let (_, errors) = store
.delete_objects(
bucket,
objects,
ObjectOptions {
delete_recovery_work_budget: Some(Arc::clone(&budget)),
..Default::default()
},
)
.await;
assert!(
errors
.iter()
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
"all pools and sets must share the finite allowance"
);
assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
assert!(
dirs.iter()
.all(|dir| dir.path().join(bucket).join(&keys[0]).join("child-7").is_dir())
);
}
#[tokio::test]
async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
let bucket = "recover-bounded-fast-commit";
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
// Inline cleanup needs no directory enumeration and may legitimately
// finish with zero scan credits. Require real part files here.
for disk in &disks {
let info = disk
.read_version("", bucket, "object", "", &ReadOptions::default())
.await
.expect("read non-inline source");
assert!(!info.inline_data(), "the fixture must require physical part cleanup");
let data_dir = info.data_dir.expect("physical data directory");
let part = std::path::PathBuf::from(disk.endpoint().to_string())
.join(bucket)
.join("object")
.join(data_dir.to_string())
.join("part.1");
assert!(part.is_file(), "the cleanup scan must encounter a real part");
}
store
.delete_object(
bucket,
"object",
ObjectOptions {
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
..Default::default()
},
)
.await
.expect("a cached prepared receipt commits independently of cleanup budget");
assert!(
store
.ctx
.delete_recovery_hints()
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "object".to_owned(),
incarnation: Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation")
),
})
);
assert!(
!disks[0]
.discover_delete_transactions(bucket, Some("object"), None, 128)
.await
.expect("retained recovery proof")
.transactions
.is_empty()
);
store
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
.await
.expect("background finishes committed cleanup");
}
#[tokio::test]
async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
let bucket = "recover-bounded-candidate";
let (dirs, store, _) = fixture(bucket).await;
let transaction = Uuid::new_v4();
for directory in &dirs {
let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
std::fs::create_dir_all(&candidate).expect("candidate path");
std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
}
assert!(
store
.delete_object(bucket, "candidate", ObjectOptions::default())
.await
.is_err()
);
for incarnation in [
None,
Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation"),
),
] {
assert!(
!store
.ctx
.delete_recovery_hints()
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "candidate".to_owned(),
incarnation,
})
);
}
}
#[tokio::test]
async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
let bucket = "recover-bounded-evidence";
let (_dirs, store, disks) = fixture(bucket).await;
let payload = vec![71; 4096];
put(&store, bucket, &payload, false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let before = disks[0]
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("before refusal");
let error = store
.delete_object(
bucket,
"object",
ObjectOptions {
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
..Default::default()
},
)
.await
.expect_err("no incomplete recovery admission");
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
let after = disks[0]
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("after refusal");
assert_eq!(before.fingerprint, after.fingerprint);
store
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
.await
.expect("background-compatible recovery");
assert_eq!(body(&store, bucket, None).await, payload);
}
#[tokio::test]
async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
let bucket = "recover-bounded-commit";
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &vec![72; 4096], false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let set = &store.pools[0].disk_set[0];
let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "object".to_owned(),
incarnation: None,
};
let ticket = set.ctx.delete_recovery_hints().register(key.clone());
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
// This hook disables local fast cleanup and runs only after receipt
// publication and the shared recovery path's key-wide GC inspection.
// Exhausting preparation instead would correctly reject the delete.
test_hooks::before_gc_finalize(bucket, "object", {
let budget = Arc::clone(&budget);
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
Arc::new(move || {
let budget = Arc::clone(&budget);
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
Box::pin(async move {
assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
})
})
});
set.publish_delete_commit(
bucket,
"object",
transaction,
&set.disk_inventory().await,
set.set_drive_count / 2 + 1,
DeleteRecoveryContext {
work_budget: Some(Arc::clone(&budget)),
recovery_ticket: ticket,
..DeleteRecoveryContext::new(None, &|| true)
},
)
.await
.expect("the durable decision still succeeds");
assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
assert_eq!(budget.remaining(), 0);
assert!(set.ctx.delete_recovery_hints().is_pending(&key));
let state = disks[0]
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("retained decision");
assert!(receipt(state.state));
store
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
.await
.expect("eventual cleanup");
assert!(
disks[0]
.discover_delete_transactions(bucket, Some("object"), None, 128)
.await
.expect("cleaned proofs")
.transactions
.is_empty()
);
}
#[tokio::test]
async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
let bucket = "recover-undecided";
+493 -9
View File
@@ -34,6 +34,7 @@ mod heal_lock_wait_tests;
#[derive(Clone, Copy)]
struct ExplicitVersionHeal<'a> {
opts: &'a HealOpts,
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
allow_regeneration: bool,
preserve_unrepairable: bool,
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
@@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> {
retirement: Option<&'a MarkerRetirementContext<'a>>,
}
async fn restore_heal_exclusive_guard(
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
) -> disk::error::Result<()> {
tokio::time::timeout(get_lock_acquire_timeout(), async {
loop {
{
let mut guard = guard.lock().await;
if guard.is_lock_lost() {
return Err(DiskError::DiskOngoingReq);
}
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
if !matches!(waiting, Ok(Some(false))) {
return Err(DiskError::Timeout);
}
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
.await
.unwrap_or(false)
{
// A writer may have waited on a peer that upgraded while
// another peer was still waiting for its readers to drain.
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
return if matches!(waiting, Ok(Some(false))) {
Ok(())
} else {
Err(DiskError::Timeout)
};
}
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.map_err(|_| DiskError::Timeout)?
}
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
loop {
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let waiting =
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
if !matches!(waiting, Ok(Some(false))) {
return;
}
}
}
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
drop(tokio::spawn(async move {
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
}));
}
// Keep the large heal state machine and its construction temporary out of
// each dispatch caller's poll frame. Ordinary object writes do not use this
// heal-only allocation boundary.
@@ -844,6 +897,7 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts,
staging_guard: None,
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: None,
@@ -994,6 +1048,7 @@ impl SetDisks {
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
let ExplicitVersionHeal {
opts,
staging_guard,
allow_regeneration: allow_explicit_version_regen,
preserve_unrepairable,
commit_fence,
@@ -1701,7 +1756,29 @@ impl SetDisks {
let dst_data_dir = data_dir;
if !latest_meta.deleted && !latest_meta.is_remote() {
let stage_result: disk::error::Result<Option<DiskError>> = async {
let mut shared_staging = false;
if let Some(guard) = staging_guard {
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
guard.lock().await.supports_heal_lock_transition().await
})
.await
.unwrap_or(false);
if supported {
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
guard.lock().await.downgrade_committed_tail().await
})
.await
.unwrap_or(false);
if converted {
shared_staging = true;
} else {
// Some peers may have converted. Abort without committing;
// dropping the guard releases each retained lease.
return Err(DiskError::Timeout);
}
}
}
let stage = async {
let erasure_info = latest_meta.erasure.clone();
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
@@ -1926,17 +2003,43 @@ impl SetDisks {
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
Ok(None)
}
.await;
};
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
if let Some(guard) = staging_guard {
tokio::select! {
result = stage => result,
_ = wait_for_heal_writer(guard) => {
// The UUID prefix is private to this attempt. Cleanup must not
// retain the namespace lock while a foreground writer waits;
// the stale-tmp sweeper also covers a cancelled disk operation.
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
return Err(DiskError::Timeout);
}
}
} else {
stage.await
}
} else {
stage.await
};
match stage_result {
Ok(None) => {}
failure => {
// Release readers and writers before removing all parts
// staged under this heal's exclusively owned prefix.
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
if shared_staging {
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
} else {
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
}
return failure.map(|error| (result, error));
}
}
if shared_staging
&& let Some(guard) = staging_guard
&& let Err(error) = restore_heal_exclusive_guard(guard).await
{
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
return Err(error);
}
}
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
if _read_repair_read_lock_guard
@@ -3616,6 +3719,7 @@ impl SetDisks {
&version,
ExplicitVersionHeal {
opts: &opts,
staging_guard: None,
allow_regeneration: false,
preserve_unrepairable: true,
commit_fence: Some(can_commit),
@@ -3676,6 +3780,7 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &read_opts,
staging_guard: None,
allow_regeneration: false,
preserve_unrepairable: true,
commit_fence: Some(&can_commit),
@@ -3702,7 +3807,7 @@ impl SetDisks {
.new_ns_lock(bucket, object)
.await
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
Some(Arc::new(
Some(Arc::new(tokio::sync::Mutex::new(
measure_heal_object_lock(
"heal_object",
HealObjectLockKind::Write,
@@ -3714,7 +3819,7 @@ impl SetDisks {
.narrow_to_disk()
.unwrap_or_else(DiskError::other)
})?,
))
)))
} else {
None
};
@@ -3732,10 +3837,18 @@ impl SetDisks {
} else {
None
};
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
guard.lock().await.lock_lost_signal()
} else {
None
};
let locks_lost = || {
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
};
// Recovery cleanup can outlive its future. Never permit reads until
// every physical operation holding its recovery owner has finished.
let recovery_tail = Arc::new(());
let mut recovery_removed_version = false;
if !opts.read_repair
&& _write_lock_guard.is_some()
@@ -3762,6 +3875,7 @@ impl SetDisks {
Arc::new((
scope.clone(),
self.ctx.begin_namespace_commit(),
Arc::clone(&recovery_tail),
_write_lock_guard.clone(),
transaction_guard.clone(),
)) as Arc<dyn Send + Sync>
@@ -3885,6 +3999,9 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &inner_opts,
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
.then_some(_write_lock_guard.as_ref())
.flatten(),
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&commit_fence),
@@ -3909,6 +4026,9 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &inner_opts,
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
.then_some(_write_lock_guard.as_ref())
.flatten(),
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&commit_fence),
@@ -6047,6 +6167,7 @@ mod heal_result_report_tests {
"",
ExplicitVersionHeal {
opts: &opts,
staging_guard: None,
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&can_commit),
@@ -10637,6 +10758,369 @@ mod heal_result_report_tests {
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-storage-entry";
let object = "object.bin";
let expected = vec![0x37; 1024 * 1024];
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(expected.clone()),
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write object");
let metadata = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external object data").to_string())
.join("part.1");
tokio::fs::remove_file(&missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal = tokio::spawn(async move {
let mut absence = None;
let result = heal_set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
read_repair: false,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await;
(result, absence)
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("read-repair reached shard reconstruction");
let mut reader = tokio::time::timeout(
Duration::from_millis(200),
set.get_object_reader(
bucket,
object,
None,
http::HeaderMap::new(),
&ObjectOptions {
suppress_read_repair: true,
..Default::default()
},
),
)
.await
.expect("background heal reconstruction should allow concurrent readers")
.expect("open object while reconstruction is paused");
let mut during_heal = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
.await
.expect("read completes while reconstruction is paused")
.expect("read object during reconstruction");
assert_eq!(during_heal, expected);
drop(reader);
release.notify_one();
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("repair finishes")
.expect("repair task joins");
let (_, error) = result.expect("repair operation succeeds");
assert!(error.is_none(), "repair failed: {error:?}");
assert!(absence.is_none());
assert!(missing.exists(), "missing shard was not restored");
let mut reader = set
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read repaired object");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read full object");
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-long-put";
let object = "object.bin";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original object");
let metadata = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external data").to_string())
.join("part.1");
tokio::fs::remove_file(missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal = tokio::spawn(async move {
let mut absence = None;
heal_set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("reconstruction started");
let put = tokio::time::timeout(
Duration::from_secs(7),
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
)
.await;
release.notify_one();
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("heal finishes after releasing reconstruction")
.expect("heal task joins");
put.expect("foreground PUT finishes during reconstruction")
.expect("foreground PUT must not time out");
assert!(
matches!(heal_result, Err(Error::Timeout)),
"interrupted heal must be retryable: {heal_result:?}"
);
drop(_pause);
let mut absence = None;
let (_, retry_error) = set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
.expect("a later heal must retry the current object");
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
let mut reader = set
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read foreground write after heal");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read full object");
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-version-delete";
let object = "object.bin";
set.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let original = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
&ObjectOptions {
versioned: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original version");
let version = original.version_id.expect("version id").to_string();
let survivor = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x52; 1024]),
&ObjectOptions {
versioned: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write surviving version")
.version_id
.expect("surviving version id");
let metadata = disks[0]
.read_version("", bucket, object, &version, &ReadOptions::default())
.await
.expect("read original metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external data").to_string())
.join("part.1");
tokio::fs::remove_file(missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal_version = version.clone();
let heal = tokio::spawn(async move {
heal_set
.heal_object_with_retirement(
bucket,
object,
&heal_version,
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut None,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("reconstruction started");
let delete = tokio::time::timeout(
Duration::from_secs(7),
set.delete_object(
bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
),
)
.await;
release.notify_one();
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("heal finishes")
.expect("heal task joins");
delete
.expect("foreground delete finishes")
.expect("foreground delete succeeds");
assert!(
matches!(heal_result, Err(Error::Timeout)),
"interrupted heal must be retryable: {heal_result:?}"
);
let result = set
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
..Default::default()
},
)
.await;
assert!(
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
"stale heal must not resurrect the deleted version: {result:?}"
);
let mut survivor = set
.get_object_reader(
bucket,
object,
None,
http::HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(survivor.to_string()),
..Default::default()
},
)
.await
.expect("surviving version remains readable");
let mut body = Vec::new();
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
assert_eq!(body, vec![0x52; 1024]);
for disk in &disks {
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
assert!(
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"deleted version must remain absent on each disk: {physical:?}"
);
}
}
#[tokio::test]
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
+6 -89
View File
@@ -95,8 +95,13 @@ impl SetDisks {
bucket: &str,
object: &str,
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
let resource = ObjectKey::with_version(
bucket,
object_name_disk_identity(object).as_ref(),
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
);
let guard = self
.namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
.namespace_lock_for_key(resource)
.await
.get_write_lock(super::super::get_lock_acquire_timeout())
.await
@@ -106,94 +111,6 @@ impl SetDisks {
Ok(Arc::new(guard))
}
fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
ObjectKey::with_version(
bucket,
object_name_disk_identity(object).as_ref(),
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
)
}
/// Batched write-lock acquisition for the whole key set of one bulk
/// mutation: one lock round instead of one quorum round-trip per key.
///
/// Bulk delete is the only multi-key write-lock holder. The batched round
/// is deadlock-free by construction (a single `lock_batch` RPC per lock
/// client, and every lock server orders concurrent batches by the same
/// global (shard, key) order), so unlike an optimistic probe wave it
/// needs no canonical serial fallback. Outcomes are per key and use the
/// same error shape as the single-key paths.
async fn namespace_write_locks_batch(
&self,
bucket: &str,
resources: &[ObjectKey],
timeout: std::time::Duration,
mode: &'static str,
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
if resources.is_empty() {
return Vec::new();
}
let lock = self.namespace_lock_for_key(resources[0].clone()).await;
lock.get_write_locks_batch(resources, timeout)
.await
.into_iter()
.zip(resources)
.map(|(result, resource)| {
result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
})
.collect()
}
/// Batched raw namespace write locks for a bulk delete's key set. The
/// resource keys use the same disk-identity folding as `new_ns_lock`, so
/// aliases (`a\b` vs `a/b`) still share one lock.
pub(crate) async fn acquire_object_write_locks_batch(
&self,
bucket: &str,
objects: &[&str],
timeout: std::time::Duration,
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
let resources: Vec<ObjectKey> = objects
.iter()
.map(|object| ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object_name_disk_identity(object).as_ref()),
version: None,
})
.collect();
self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
}
/// Batched delete-transaction locks for a bulk delete's key set; see
/// [`Self::acquire_delete_transaction_lock`] for the resource shape.
pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
&self,
bucket: &str,
objects: &[&str],
timeout: std::time::Duration,
) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
let resources: Vec<ObjectKey> = objects
.iter()
.map(|object| self.delete_transaction_resource(bucket, object))
.collect();
let mut guards = Vec::with_capacity(resources.len());
for (object, result) in objects.iter().zip(
self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
.await,
) {
match result {
Ok(guard) => {
#[cfg(test)]
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
let _ = object;
guards.push(Ok(Arc::new(guard)));
}
Err(error) => guards.push(Err(error)),
}
}
guards
}
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
match err {
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -165,6 +165,7 @@ impl SetDisks {
bucket,
object,
opts.no_lock,
false,
DecommissionCapacityAdmission::Mutation,
)
.await?
@@ -275,6 +276,7 @@ impl SetDisks {
bucket,
object,
opts.no_lock,
false,
DecommissionCapacityAdmission::Mutation,
)
.await?
@@ -52,8 +52,8 @@ pub(crate) mod list {
pub(crate) mod multipart {
pub(crate) use rustfs_storage_api::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
MultipartUploadInspection, MultipartUploadResult, PartInfo,
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
MultipartUploadResult, PartInfo,
};
}
+46 -292
View File
@@ -14,7 +14,7 @@
use super::*;
use crate::bucket::{
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState},
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState, table_bucket_catalog_metadata_prefix},
utils::is_meta_bucketname,
};
use crate::error::is_err_bucket_not_found;
@@ -145,41 +145,11 @@ async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext
}
}
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
use crate::bucket::metadata::*;
let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
if bucket != "ahm-replacement" {
prefixes.push(root);
return prefixes;
}
// This valid user bucket shares its metadata directory with raw drive
// replacement intents, seals and recovery proofs. Delete only bucket-owned
// paths, including legacy standalone configuration, never the shared root.
// S3 objects live in the separate user volume; configuration APIs write the
// aggregate metadata file, so clients cannot populate arbitrary siblings.
prefixes.extend(
[
BUCKET_METADATA_FILE,
BUCKET_INCARNATION_FILE,
BUCKET_POLICY_CONFIG,
BUCKET_QUOTA_CONFIG_FILE,
BUCKET_TARGETS_FILE,
BUCKET_ACL_CONFIG,
BUCKET_TABLE_CONFIG,
BUCKET_DURABILITY_CONFIG,
BUCKET_ON_DEMAND_MIGRATION_CONFIG,
".replication",
"integrity-jobs",
crate::data_usage::DATA_USAGE_CACHE_NAME,
]
.into_iter()
.chain(XML_BUCKET_CONFIG_FILES)
.map(|file| format!("{root}/{file}")),
);
prefixes
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
[
table_bucket_catalog_metadata_prefix(bucket),
format!("{BUCKET_META_PREFIX}/{bucket}"),
]
}
fn bucket_deleted_marker_prefix(bucket: &str) -> String {
@@ -190,48 +160,41 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
}
// Box the child before constructing either lease wrapper. Keeping F inline in
// nested select futures multiplies poll-frame temporaries along DELETE -> quota
// cleanup -> metadata GET. This boundary must not inline back into that chain.
#[inline(never)]
pub(crate) fn await_bucket_namespace_operation<'a, T: 'a, F>(
guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
bucket: &'a str,
pub(crate) async fn await_bucket_namespace_operation<T, F>(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> impl Future<Output = Result<T>> + 'a
) -> Result<T>
where
F: Future<Output = Result<T>> + 'a,
F: Future<Output = Result<T>>,
{
let future = Box::pin(future);
async move {
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
}
fn await_bucket_lifecycle_operation<'a, T: 'a, F>(
lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
namespace_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
bucket: &'a str,
async fn await_bucket_lifecycle_operation<T, F>(
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> impl Future<Output = Result<T>> + 'a
) -> Result<T>
where
F: Future<Output = Result<T>> + 'a,
F: Future<Output = Result<T>>,
{
await_bucket_namespace_operation(
lifecycle_guard,
@@ -239,6 +202,7 @@ where
operation,
await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
)
.await
}
fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
@@ -514,18 +478,10 @@ impl ECStore {
let mut tags = HashMap::with_capacity(tagging.tag_set.len());
for tag in &tagging.tag_set {
let (Some(key), Some(value)) = (&tag.key, &tag.value) else {
return Err(crate::bucket::metadata::unreadable_config_error(
bucket,
BUCKET_TAGGING_CONFIG,
metadata.tagging_config_xml.len(),
));
return Err(Error::other("stored bucket tag is missing a key or value"));
};
if key.is_empty() || tags.insert(key.clone(), value.clone()).is_some() {
return Err(crate::bucket::metadata::unreadable_config_error(
bucket,
BUCKET_TAGGING_CONFIG,
metadata.tagging_config_xml.len(),
));
return Err(Error::other("stored bucket tags contain empty or duplicate keys"));
}
}
Ok(tags)
@@ -1717,17 +1673,7 @@ impl ECStore {
ns_guard.as_ref(),
bucket,
"bucket retirement publication",
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
)
.await?;
// The lifecycle fence excludes live reservation owners. A crash-left
// V1 ledger must not block quota admission after same-name recreation.
await_bucket_lifecycle_operation(
bucket_lifecycle_guard.as_ref(),
ns_guard.as_ref(),
bucket,
"retired bucket quota ledger cleanup",
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
)
.await?;
}
@@ -1878,20 +1824,6 @@ mod tests {
assert!(!xlmeta.diagnostic_truncated);
}
#[tokio::test]
async fn bucket_namespace_operation_keeps_nested_future_state_off_the_poll_stack() {
let payload = std::hint::black_box([7u8; 128 * 1024]);
let operation = super::await_bucket_lifecycle_operation(None, None, "bucket", "large metadata operation", async move {
tokio::task::yield_now().await;
Ok(std::hint::black_box(payload)[0])
});
assert!(
std::mem::size_of_val(&operation) < 1024,
"nested lease wrappers must not retain the concrete storage future inline"
);
assert_eq!(operation.await.expect("the boxed operation must still execute"), 7);
}
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
@@ -2308,13 +2240,7 @@ mod tests {
let result = store.get_bucket_tags_for_policy(bucket).await;
match expected {
Some(tags) => assert_eq!(&result.expect("valid tag configuration"), tags),
None => assert!(
crate::bucket::metadata::unreadable_config_refusal(
&result.expect_err("corrupt tags must not become an untagged bucket")
)
.is_some_and(|failure| failure.config_file == super::BUCKET_TAGGING_CONFIG),
"corrupt tags must carry a typed tagging refusal: {xml:?}"
),
None => assert!(result.is_err(), "corrupt tags must not become an untagged bucket: {xml:?}"),
}
}
}
@@ -2368,8 +2294,17 @@ mod tests {
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("fixture rename tail should finish before checking its generation"),
.expect("fixture object lock should be available"),
);
// A quorum-acknowledged rename can release the object lock while its
// physical tail still holds the instance-wide namespace owner.
tokio::time::timeout(Duration::from_secs(30), async {
while ecstore.ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("fixture namespace owners should drain before checking its generation");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(3),
@@ -2516,187 +2451,6 @@ mod tests {
assert!(prefixes.contains(&"buckets/analytics".to_string()));
}
#[tokio::test]
#[serial]
async fn replacement_named_bucket_delete_preserves_recovery_and_recreates_clean_metadata() {
use crate::bucket::metadata::{BUCKET_QUOTA_CONFIG_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
let (_temp, store) = setup_bucket_quorum_test_env(&[4], None).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "ahm-replacement";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create legal bucket");
let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
for (config, bytes) in [
(BUCKET_QUOTA_CONFIG_FILE, br#"{"quota":1024,"quota_type":"Hard"}"#.as_slice()),
(
BUCKET_TAGGING_CONFIG,
b"<Tagging><TagSet><Tag><Key>old</Key><Value>value</Value></Tag></TagSet></Tagging>".as_slice(),
),
(
BUCKET_VERSIONING_CONFIG,
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".as_slice(),
),
] {
metadata_sys::update_in(&store.ctx, bucket, config, bytes.to_vec())
.await
.expect("persist old configuration");
}
let ledger_path = format!("config/quota-ledger/{bucket}.json");
// V1 precharges outstanding growth: four live bytes plus the
// crash-left reservation's 100 bytes must already be accounted.
let ledger = serde_json::to_vec(&serde_json::json!({
"version": 1, "bucket_incarnation": old_incarnation,
"quota_revision_unix_nanos": 0, "accounted_usage": 104,
"reservations": {Uuid::new_v4().to_string(): {
"object": "crashed-put", "old_size": 0, "new_size": 100,
"created_at": 0, "pool_index": 0, "set_index": 0, "commit_started": false
}}, "reconcile_required": true
}))
.expect("encode crash-left ledger");
// The storage API requires the same legacy-quota admission supplied
// by the S3 handler, even for this small fixture object.
let mut live_options = ObjectOptions {
versioned: true,
..Default::default()
};
assert!(live_options.set_quota_admission(0, 1024));
// Failed deletion must leave both the live incarnation and its ledger alone.
store
.put_object(bucket, "live", &mut PutObjReader::from_vec(b"live".to_vec()), &live_options)
.await
.expect("write live object");
crate::config::com::save_config(store.clone(), &ledger_path, ledger.clone())
.await
.expect("persist old ledger");
assert!(matches!(
store.delete_bucket(bucket, &DeleteBucketOptions::default()).await,
Err(StorageError::BucketNotEmpty(_))
));
assert_eq!(
crate::config::com::read_config(store.clone(), &ledger_path)
.await
.expect("live ledger remains"),
ledger
);
// Remove every version; the configured bucket is otherwise still empty.
let versions = store
.clone()
.list_object_versions(bucket, "", None, None, None, 100)
.await
.expect("list live versions");
for object in versions.objects {
store
.delete_object(
bucket,
&object.name,
ObjectOptions {
version_id: object.version_id.map(|id| id.to_string()),
..Default::default()
},
)
.await
.expect("remove live version");
}
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
let task = Uuid::new_v4();
let recovery_names = [
"ahm_replacement_intent.json",
"ahm_replacement_intent_seal",
"ahm_replacement_completion_proof.json",
]
.map(|suffix| format!("{task}_{suffix}"));
let disk_paths = store.pools[0].disk_set[0]
.disks
.read()
.await
.iter()
.flatten()
.map(|disk| PathBuf::from(disk.endpoint().to_string()))
.collect::<Vec<_>>();
for disk in &disk_paths {
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
for name in &recovery_names {
tokio::fs::write(directory.join(name), b"durable heal record")
.await
.expect("seed raw recovery record");
}
// Legacy standalone settings and bucket-owned subtrees must not survive.
for name in [
"quota.json",
"versioning.xml",
"policy.json",
"bucket-targets.json",
"integrity-jobs/job.json",
".replication/resync.bin",
".usage-cache.bin",
] {
let path = directory.join(name);
tokio::fs::create_dir_all(path.parent().expect("fixture parent"))
.await
.expect("legacy parent");
tokio::fs::write(path, b"old bucket configuration")
.await
.expect("seed legacy configuration");
}
}
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete legal same-name bucket");
assert!(matches!(
crate::config::com::read_config(store.clone(), &ledger_path).await,
Err(StorageError::ConfigNotFound)
));
for disk in &disk_paths {
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
for name in &recovery_names {
assert_eq!(
tokio::fs::read(directory.join(name)).await.expect("recovery record survives"),
b"durable heal record"
);
}
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket)
.into_iter()
.filter(|p| p.starts_with(&root))
{
assert!(
!disk.join(RUSTFS_META_BUCKET).join(prefix).exists(),
"bucket-owned metadata must be removed"
);
}
}
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate legal bucket");
let fresh = metadata_sys::get_in(&store.ctx, bucket).await.expect("fresh metadata");
assert_ne!(fresh.bucket_incarnation_id, old_incarnation);
assert!(fresh.quota_config_json.is_empty());
assert!(fresh.tagging_config_xml.is_empty());
assert!(fresh.versioning_config_xml.is_empty());
assert!(fresh.policy_config_json.is_empty());
// The strict purge/recovery cleanup must share the same ownership boundary.
store
.cleanup_deleted_bucket_metadata_strict(bucket, true)
.await
.expect("strict metadata cleanup");
for disk in &disk_paths {
for name in &recovery_names {
assert_eq!(
tokio::fs::read(disk.join(RUSTFS_META_BUCKET).join(&root).join(name))
.await
.expect("strict cleanup preserves recovery"),
b"durable heal record"
);
}
}
}
#[test]
fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
+139 -5
View File
@@ -45,7 +45,8 @@
use crate::error::{Result, StorageError};
use std::collections::HashMap;
use std::future::Future;
use std::sync::{Arc, Mutex};
use std::sync::{Arc, Mutex, Weak};
use std::time::Duration;
use rustfs_lock::NamespaceLockGuard;
use rustfs_lock::distributed_lock::LockLostSignal;
@@ -65,7 +66,7 @@ struct RegisteredGuard {
enum LockLossProbe {
Distributed(Arc<LockLostSignal>),
Local,
Local(Option<rustfs_lock::fast_lock::guard::FastLockGuardProbe>),
#[cfg(test)]
Test(Arc<std::sync::atomic::AtomicBool>),
}
@@ -74,14 +75,19 @@ impl LockLossProbe {
fn from_guard(guard: &NamespaceLockGuard) -> Self {
match guard.lock_lost_signal() {
Some(signal) => Self::Distributed(signal),
None => Self::Local,
None => Self::Local(match guard {
NamespaceLockGuard::Fast(guard) => guard.release_probe(),
NamespaceLockGuard::Standard(_) => None,
}),
}
}
fn is_lost(&self) -> bool {
// Registry callers hold entries before checking a local shard; shard
// operations never acquire the bucket fence registry.
match self {
Self::Distributed(signal) => signal.is_lost(),
Self::Local => false,
Self::Local(probe) => probe.as_ref().is_some_and(|probe| probe.is_lost()),
#[cfg(test)]
Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst),
}
@@ -98,9 +104,74 @@ pub(super) struct FenceRegistration {
#[derive(Default)]
pub(crate) struct BucketFenceRegistry {
entries: Mutex<HashMap<String, FenceEntry>>,
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
}
#[derive(Default)]
pub(super) struct ReservationFenceSlot {
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
}
/// Shared by short bursts of reservations, but never kept alive by the cache.
pub(super) struct ReservationFenceGuard {
publication: NamespaceLockGuard,
lifecycle: BucketIncarnationFenceGuard,
created_at: tokio::time::Instant,
_slot: Arc<ReservationFenceSlot>,
}
impl ReservationFenceGuard {
pub(super) fn new(
publication: NamespaceLockGuard,
lifecycle: BucketIncarnationFenceGuard,
slot: Arc<ReservationFenceSlot>,
) -> Self {
Self {
publication,
lifecycle,
created_at: tokio::time::Instant::now(),
_slot: slot,
}
}
pub(super) fn is_reusable(&self) -> bool {
// Stop admitting new reservations through this shared reader so a
// queued bucket writer can progress after the existing tails drain.
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
}
fn is_held(&self) -> bool {
!self.publication.is_lock_lost()
&& !self.publication.is_released()
&& !self.lifecycle.is_lock_lost()
&& self
.lifecycle
.namespace_lock_guard()
.is_some_and(|guard| !guard.is_released())
}
}
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
fn is_valid(&self) -> bool {
self.is_held()
}
}
impl BucketFenceRegistry {
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
let key = (bucket.to_owned(), incarnation);
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
return slot;
}
if slots.len() >= 1024 {
slots.retain(|_, slot| slot.strong_count() != 0);
}
let slot = Arc::new(ReservationFenceSlot::default());
slots.insert(key, Arc::downgrade(&slot));
slot
}
/// Register a new live guard for `bucket` and return the memoized
/// incarnation id if one is valid for the current coverage window.
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
@@ -234,7 +305,9 @@ impl BucketIncarnationFenceGuard {
}
pub fn is_lock_lost(&self) -> bool {
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
self.inner
.as_ref()
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
}
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
@@ -352,6 +425,67 @@ mod tests {
reg.exit("b", fourth.token, false);
}
#[tokio::test]
async fn disabled_local_lock_does_not_add_a_namespace_fence() {
let manager = Arc::new(rustfs_lock::GlobalLockManager::Disabled(
rustfs_lock::fast_lock::DisabledLockManager::new(),
));
let lock = NamespaceLock::with_local_manager("disabled-fence".into(), manager);
let guard = lock
.get_read_lock(ObjectKey::new("b", "object"), "owner", Duration::from_secs(1))
.await
.expect("disabled lock");
let mut opts = crate::object_api::ObjectOptions::default();
opts.add_namespace_lock_guard(&guard);
assert!(opts.namespace_lock_fence.is_none());
}
#[tokio::test]
async fn forced_local_fence_loss_invalidates_live_incarnation_memo() {
let manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
let global = Arc::new(rustfs_lock::GlobalLockManager::Enabled(manager.clone()));
let lock = NamespaceLock::with_local_manager("forced-fence".into(), global);
let key = ObjectKey::new("b", "lifecycle");
let registry = Arc::new(BucketFenceRegistry::default());
let inner = lock
.get_read_lock(key.clone(), "first", Duration::from_secs(1))
.await
.expect("first lock");
let mut first = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
let mut opts = crate::object_api::ObjectOptions::default();
first.attach_to_object_options(&mut opts);
opts.add_namespace_lock_guard(first.namespace_lock_guard().expect("local guard"));
assert!(
!opts
.bucket_lifecycle_lock_fence
.as_ref()
.expect("lifecycle fence")
.is_lock_lost()
);
assert!(!opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
assert_eq!(manager.force_unlock(&key), 1);
assert!(
opts.bucket_lifecycle_lock_fence
.as_ref()
.expect("attached fence")
.is_lock_lost()
);
assert!(opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
let writer = lock
.get_write_lock(key.clone(), "writer", Duration::from_secs(1))
.await
.expect("writer");
drop(writer);
let inner = lock
.get_read_lock(key, "second", Duration::from_secs(1))
.await
.expect("second lock");
let mut second = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
assert_eq!(second.validate(true, || async { Ok(uuid(8)) }).await.expect("reload"), uuid(8));
assert!(first.validate(true, || async { Ok(uuid(7)) }).await.is_err());
}
#[tokio::test]
async fn fence_guard_forwards_distributed_lock_loss_to_registry() {
let registry = Arc::new(BucketFenceRegistry::default());
+5 -311
View File
@@ -368,7 +368,7 @@ impl ECStore {
Some(expected) => expected,
None => self.bucket_incarnation_id_from_disk(bucket).await?,
};
let object = object.map(encode_dir_object);
let object = object.map(str::to_owned);
let prefix = prefix.map(str::to_owned);
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
@@ -430,7 +430,7 @@ impl ECStore {
HealItemType::Bucket
}
.to_string(),
object: object.map(|object| decode_dir_object(&object)).or(prefix).unwrap_or_default(),
object: object.or(prefix).unwrap_or_default(),
bucket,
detail: format!(
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
@@ -656,48 +656,12 @@ impl ECStore {
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
}
/// Read the published bucket identity without fabricating or migrating metadata.
/// Absence is distinct from quorum, corruption and transaction-fence failures.
pub async fn replacement_bucket_incarnation_from_disk(self: &Arc<Self>, bucket: &str) -> Result<Option<uuid::Uuid>> {
let transaction =
crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
let incarnation = crate::bucket::metadata::load_bucket_incarnation(self.clone(), bucket).await?;
if transaction.is_lock_lost() {
return Err(Error::PreconditionFailed);
}
Ok(incarnation)
}
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
pub async fn heal_replacement_bucket_metadata(
self: &Arc<Self>,
bucket: &str,
opts: &HealOpts,
targets: &[String],
) -> Result<()> {
// Reject an invalid request before touching the drives so callers keep
// the PreconditionFailed contract for bad buckets and options.
if opts.pool.is_none()
|| opts.set.is_none()
|| targets.is_empty()
|| opts.dry_run
|| opts.no_lock
|| check_valid_bucket_name_strict(bucket).is_err()
{
return Err(Error::PreconditionFailed);
}
let expected = self.bucket_incarnation_id_from_disk(bucket).await?;
self.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
.await
}
/// Keep a replacement pass bound to its admitted bucket through metadata commit.
pub async fn heal_replacement_bucket_metadata_at_incarnation(
self: &Arc<Self>,
bucket: &str,
expected: uuid::Uuid,
opts: &HealOpts,
targets: &[String],
) -> Result<()> {
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
@@ -734,8 +698,9 @@ impl ECStore {
// Read the persisted identity without lazy migration: a replacement
// must never turn lost configuration into newly fabricated defaults.
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
let targets = targets.to_vec();
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, opts| async move {
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
// Match config writers: lifecycle, metadata transaction, object locks.
// Keep the transaction stable through target readback, including when
// the caller is cancelled and the storage owner finishes its write.
@@ -1057,7 +1022,6 @@ impl ECStore {
set_index: usize,
targets: &[String],
) -> Result<bool> {
let object = encode_dir_object(object);
let pool = self
.pools
.get(pool_index)
@@ -1070,7 +1034,7 @@ impl ECStore {
)
})?;
set.replacement_targets_have_version(bucket, &object, version_id, targets)
set.replacement_targets_have_version(bucket, object, version_id, targets)
.await
.map_err(Into::into)
}
@@ -2895,105 +2859,6 @@ mod tests {
}
}
#[tokio::test]
#[serial_test::serial]
async fn replacement_directory_object_readback_and_exact_recovery_use_encoded_key() {
let (_root, store, shutdown) = heal_store_with_pool_count(1).await;
let bucket = "directory-replacement-readback";
let object = "folder/";
let encoded = encode_dir_object(object);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
store
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(Vec::new()),
&ObjectOptions {
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write directory object through the public store entry");
let set = &store.pools[0].disk_set[0];
let targets = vec![set.set_endpoints[0].to_string()];
assert!(
store
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
.await
.expect("readback")
);
let disk = set.disks.read().await[0].clone().expect("target disk");
disk.delete(
bucket,
&encoded,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("remove target directory shard");
assert!(
!store
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
.await
.expect("missing target")
);
let opts = HealOpts {
pool: Some(0),
set: Some(0),
..Default::default()
};
let (result, error) = store
.heal_object(bucket, object, "", &opts)
.await
.expect("heal directory object");
assert!(error.is_none(), "heal failed: {error:?}");
assert_eq!(result.object, object);
assert!(
store
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
.await
.expect("healed readback")
);
assert!(
store
.replacement_targets_have_version(bucket, &encoded, "", 0, 0, &targets)
.await
.expect("encoded readback")
);
// Exact-object recovery must inspect the same physical namespace and
// reject corrupt proof rather than silently report no transactions.
let metadata = disk
.read_all(bucket, &format!("{encoded}/xl.meta"))
.await
.expect("healed metadata");
disk.write_all(
bucket,
&format!("{encoded}/{}/delete-prepared", Uuid::new_v4()),
b"invalid proof".to_vec().into(),
)
.await
.expect("seed corrupt recovery proof");
store
.recover_delete_transactions_at_incarnation(bucket, Some(object), None, &HealOpts { dry_run: true, ..opts })
.await
.expect_err("directory recovery must discover and reject the corrupt proof");
assert_eq!(
disk.read_all(bucket, &format!("{encoded}/xl.meta"))
.await
.expect("unchanged metadata"),
metadata
);
shutdown.cancel();
}
#[tokio::test]
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
let (root, store, shutdown) = multi_pool_heal_store().await;
@@ -3827,177 +3692,6 @@ mod tests {
);
}
// The RPC lock service uses the raw ObjectKey and one backend per node,
// even when the requesting sets carry different pool namespace strings.
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
use rustfs_lock::client::{LockClient, local::LocalClient};
let clients = || {
(0..2)
.map(|_| {
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
})
.collect::<Vec<_>>()
};
let first = clients();
let second = if shared { first.clone() } else { clients() };
let mut pools = source.pools.clone();
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
for set in &mut Arc::make_mut(pool).disk_set {
Arc::make_mut(set).lockers = lockers.clone();
}
}
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
Arc::new(ECStore {
id: Uuid::new_v4(),
disk_map: Default::default(),
pools,
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
rebalance_meta: tokio::sync::RwLock::new(None),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(state),
ctx: Arc::clone(&source.ctx),
bucket_fence_registry: Arc::default(),
})
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
let first_lock = first
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("first pool lock");
let second_lock = second
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("second pool lock");
let guard = first_lock
.get_write_lock(std::time::Duration::from_secs(2))
.await
.expect("first writer");
assert!(
second_lock
.get_write_lock(std::time::Duration::from_millis(150))
.await
.is_err(),
"different pool namespaces still conflict on the same backend and object key"
);
drop(guard);
second_lock
.get_write_lock(std::time::Duration::from_secs(3))
.await
.expect("second writer should acquire after the first releases");
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let (item, error) = tokio::time::timeout(
std::time::Duration::from_secs(3),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
dry_run: true,
..Default::default()
},
),
)
.await
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
.expect("pool metadata heal inspection should return a result");
assert!(error.is_none(), "inspection should succeed: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let missing = remove_pool_meta_shard(&store, 1).await;
let (item, error) = tokio::time::timeout(
std::time::Duration::from_secs(10),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
..Default::default()
},
),
)
.await
.expect("cross-pool repair must not reenter the shared writer")
.expect("pool metadata repair should return a result");
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
assert!(
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
"the selected missing shard must be restored"
);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
let lock = target
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("target lock");
let guard = lock
.get_write_lock(std::time::Duration::from_secs(2))
.await
.expect("hold target lock");
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
let heal_store = Arc::clone(&store);
let mut heal = tokio::spawn(async move {
heal_store
.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
dry_run: true,
..Default::default()
},
)
.await
});
tokio::time::timeout(
std::time::Duration::from_secs(3),
barrier.wait_until_external_heal_target_lock_attempted(),
)
.await
.expect("inspection must attempt the independent target lock");
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
drop(guard);
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
.await
.expect("inspection should progress after target lock release")
.expect("heal task should not panic")
.expect("pool metadata heal inspection should return a result");
assert!(error.is_none(), "inspection should succeed: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
-298
View File
@@ -5155,11 +5155,6 @@ mod tests {
let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
if !sharded {
crate::bucket::quota::reservation::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
.await
.expect("bootstrap quota with one disk offline");
}
let payload = vec![0x51; 4096];
let mut reader = PutObjReader::from_vec(payload.clone());
@@ -21012,299 +21007,6 @@ mod tests {
);
}
async fn seed_retained_null_version(
store: &Arc<crate::store::ECStore>,
bucket: &str,
object: &str,
protection: &str,
latest_is_marker: bool,
) {
let metadata = if protection == "legal-hold" {
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
} else {
HashMap::from([
("x-amz-object-lock-mode".to_string(), protection.to_string()),
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
])
};
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
store
.put_object(
bucket,
object,
&mut original,
&ObjectOptions {
user_defined: metadata,
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
..Default::default()
},
)
.await
.expect("seed retained null version");
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
store
.put_object(
bucket,
object,
&mut latest,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed newer unprotected UUID version");
if latest_is_marker {
store
.delete_object(
bucket,
object,
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("create a UUID delete marker above the retained null version");
}
}
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
let mut reader = store
.get_object_reader(
bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
},
)
.await
.expect("retained null version must remain readable");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("read retained null version");
assert_eq!(body, b"protected null version");
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn null_version_object_lock_delete_checks_actual_target() {
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
let temp = tempfile::tempdir().expect("create null-version delete store");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("create Object Lock bucket");
let configs = [
VersioningConfiguration::default(),
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
];
for (case, config) in configs.into_iter().enumerate() {
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
for latest_is_marker in [false, true] {
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
let (_, errors) = store
.delete_objects(
&bucket,
vec![ObjectToDelete {
object_name: object.clone(),
..Default::default()
}],
ObjectOptions {
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
..Default::default()
},
)
.await;
assert!(
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
"batch delete must judge the retained null version: {object}: {errors:?}"
);
let error = store
.delete_object(
&bucket,
&object,
ObjectOptions {
version_suspended: true,
..Default::default()
},
)
.await
.expect_err("suspended single delete must also judge the retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
assert_retained_null_version_body(&store, &bucket, &object).await;
if protection == "GOVERNANCE" {
let (_, errors) = store
.delete_objects(
&bucket,
vec![ObjectToDelete {
object_name: object.clone(),
..Default::default()
}],
ObjectOptions {
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
bypass_governance: true,
}),
..Default::default()
},
)
.await;
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
let removed = store
.get_object_info(
&bucket,
&object,
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
},
)
.await;
match removed {
Ok(marker) => {
assert!(marker.delete_marker, "only a null delete marker may replace the payload");
assert_eq!(marker.size, 0);
assert!(marker.data_dir.is_none());
}
Err(error) => assert!(
matches!(
error,
StorageError::MethodNotAllowed
| StorageError::ObjectNotFound(_, _)
| StorageError::VersionNotFound(_, _, _)
),
"unexpected bypass result: {error:?}"
),
}
assert!(
store
.get_object_reader(
&bucket,
&object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
}
)
.await
.is_err(),
"authorized deletion must make the retained null payload unreadable"
);
}
}
}
}
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
let temp = tempfile::tempdir().expect("create null-version overwrite store");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("create Object Lock bucket");
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
for latest_is_marker in [false, true] {
let object = format!("{protection}-{latest_is_marker}");
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
let error = store
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
.await
.expect_err("PUT must reject replacement of a retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
let upload = store
.clone()
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
.await
.expect("initiate multipart upload");
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
let part = store
.clone()
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
.await
.expect("stage multipart replacement");
let error = store
.clone()
.complete_multipart_upload(
&bucket,
&object,
&upload.upload_id,
vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
.expect_err("multipart completion must reject replacement of a retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
assert_retained_null_version_body(&store, &bucket, &object).await;
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
store
.put_object(
&bucket,
&object,
&mut next_version,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("a new UUID version must remain allowed");
assert_retained_null_version_body(&store, &bucket, &object).await;
}
}
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
File diff suppressed because it is too large Load Diff
+3
View File
@@ -423,6 +423,8 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
!is_meta_bucketname(&oi.bucket)
}
const MAX_UPLOADS_LIST: usize = 10000;
mod bucket;
mod bucket_fence;
pub(crate) use bucket::await_bucket_namespace_operation;
@@ -454,6 +456,7 @@ mod rebalance;
pub(crate) use rebalance::PoolSpaceForTest;
pub(crate) mod utils;
pub(crate) use peer::admit_local_disk;
use peer::init_local_peer;
pub use peer::{
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
+15 -422
View File
@@ -273,28 +273,6 @@ async fn pause_data_movement_multipart_before_selected_completion(bucket: &str)
}
}
async fn pool_has_multipart_upload_for_key(
pool: &crate::core::sets::Sets,
bucket: &str,
object: &str,
expected_incarnation_id: Option<Uuid>,
) -> Result<bool> {
let results = stream::iter(pool.disk_set.iter().cloned())
.map(|set| async move {
set.has_multipart_upload_for_key(bucket, object, expected_incarnation_id)
.await
})
.buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY)
.collect::<Vec<_>>()
.await;
let mut found = false;
for result in results {
// Preserve errors from every set instead of hiding them behind a hit.
found |= result?;
}
Ok(found)
}
async fn list_pool_multipart_uploads_for_incarnation(
pool: &crate::core::sets::Sets,
bucket: &str,
@@ -512,8 +490,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ListPartsInfo> {
check_list_parts_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
@@ -644,8 +620,6 @@ impl ECStore {
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
check_new_object_args(bucket, object, opts.data_movement)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if self.single_pool() {
@@ -701,7 +675,21 @@ impl ECStore {
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
continue;
}
if pool_has_multipart_upload_for_key(pool, bucket, object, opts.expected_bucket_incarnation_id).await? {
let res = list_pool_multipart_uploads_for_incarnation(
pool,
bucket,
&MultipartUploadListRequest {
prefix: object.to_string(),
key_marker: None,
upload_id_marker: None,
delimiter: None,
max_uploads: MAX_UPLOADS_LIST,
expected_incarnation_id: opts.expected_bucket_incarnation_id,
},
)
.await?;
if !res.uploads.is_empty() {
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
@@ -784,8 +772,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<PartInfo> {
check_put_object_part_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if let Some(guard) = bucket_lifecycle_guard {
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
@@ -820,8 +806,6 @@ impl ECStore {
.part_number
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
check_put_object_part_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
if !opts.data_movement {
return Err(Error::other("targeted multipart upload requires data_movement options"));
}
@@ -854,8 +838,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<MultipartInfo> {
check_list_parts_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
if self.single_pool() {
@@ -889,8 +871,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<()> {
check_abort_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
for pool_idx in self.existing_multipart_pool_order().await {
@@ -1072,8 +1052,6 @@ impl ECStore {
for upload_id in upload_ids {
check_abort_multipart_args(bucket, object, upload_id)?;
}
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
if !opts.data_movement {
return Err(Error::other("targeted multipart abort requires data_movement options"));
}
@@ -1232,34 +1210,7 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
check_complete_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
// The set-level commit judges the null version this completion
// replaces; give it the same authoritative snapshot a PUT carries.
let expected_incarnation_id = opts
.expected_bucket_incarnation_id
.ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
let lifecycle_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
Some(snapshot) => Arc::clone(snapshot),
None => {
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
.await?
}
};
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
return Err(Error::other(
"Object Lock snapshot does not hold valid target bucket generation and configuration fences",
));
}
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
if !self.single_pool() {
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
}
@@ -1368,8 +1319,6 @@ impl ECStore {
let request_opts = opts;
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
if !opts.data_movement {
return Err(Error::other("targeted multipart completion requires data_movement options"));
}
@@ -1491,362 +1440,6 @@ mod tests {
};
use std::time::Duration;
#[tokio::test]
async fn create_multipart_pool_selection_uses_only_an_exact_key_upload() {
use crate::bucket::metadata_sys::init_bucket_metadata_sys;
use crate::runtime::instance::InstanceContext;
let mut dirs = Vec::new();
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let directory = tempfile::tempdir().expect("temporary disk");
let mut endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("local endpoint");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
dirs.push(directory);
endpoints.push(endpoint);
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("exact-multipart-pool-{pool_index}"),
platform: "test".to_owned(),
});
}
let endpoints = EndpointServerPools(pools);
let ctx = Arc::new(InstanceContext::new());
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
.await
.expect("initialize local disks");
let cancel = tokio_util::sync::CancellationToken::new();
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, cancel.clone(), ctx)
.await
.expect("initialize two pools");
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "exact-pool-selection";
let other_bucket = "other-pool-selection";
for bucket in [bucket, other_bucket] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
}
for (upload_bucket, key) in [(bucket, "report-long"), (bucket, "report/"), (other_bucket, "report")] {
let (opts, _guard) = store
.guard_multipart_bucket_incarnation(upload_bucket, &ObjectOptions::default())
.await
.expect("pin upload incarnation");
store.pools[0]
.new_multipart_upload(upload_bucket, &encode_dir_object(key), &opts)
.await
.expect("seed nonmatching upload in first pool");
}
let (opts, _guard) = store
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
.await
.expect("pin target incarnation");
assert!(
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", opts.expected_bucket_incarnation_id)
.await
.expect("first pool contains only other keys")
);
store.pools[1]
.new_multipart_upload(bucket, "report", &opts)
.await
.expect("seed exact upload in second pool");
let (created, pool_index, incarnation) = store
.handle_new_multipart_upload_with_pool_idx(bucket, "report", &ObjectOptions::default(), None)
.await
.expect("create another upload for the exact key");
assert_eq!(pool_index, 1, "prefix matches in the first pool must not change placement");
assert_eq!(incarnation, opts.expected_bucket_incarnation_id);
assert!(!created.upload_id.is_empty());
assert!(
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", incarnation)
.await
.expect("new upload must not appear in first pool")
);
assert!(
!pool_has_multipart_upload_for_key(&store.pools[1], bucket, "report", Some(Uuid::new_v4()))
.await
.expect("old bucket incarnations must not affect placement")
);
cancel.cancel();
}
#[tokio::test]
async fn multipart_directory_keys_preserve_the_bare_object_and_round_trip() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::object::ObjectIO as _;
use sha2::Digest;
use tokio::io::AsyncReadExt;
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
for versioned in [false, true] {
let bucket = if versioned {
"directory-mpu-versioned"
} else {
"directory-mpu-unversioned"
};
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
if versioned {
store
.update_bucket_metadata_config(
bucket,
"versioning.xml",
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec(),
)
.await
.expect("enable versioning");
}
let opts = ObjectOptions {
versioned,
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
};
let original = store
.put_object(
bucket,
"protected",
&mut PutObjReader::from_vec(b"private original".to_vec()),
&ObjectOptions {
user_defined: HashMap::from([(
rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(),
"classification=secret".to_string(),
)]),
..opts.clone()
},
)
.await
.expect("write protected bare object");
let upload = store
.new_multipart_upload(bucket, "protected/", &opts)
.await
.expect("create directory upload");
let part = store
.put_object_part(
bucket,
"protected/",
&upload.upload_id,
1,
&mut PutObjReader::from_vec(b"directory payload".to_vec()),
&opts,
)
.await
.expect("write directory part");
store
.put_object_part(
bucket,
"protected/",
&upload.upload_id,
2,
&mut PutObjReader::from_vec(b"unselected private bytes".to_vec()),
&opts,
)
.await
.expect("write an unselected part");
let upload_sha = hex_simd::encode_to_string(
sha2::Sha256::digest(format!("{bucket}/protected__XLDIR__").as_bytes()),
hex_simd::AsciiCase::Lower,
);
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
for dir in &dirs {
let staging = dir
.path()
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_sha)
.join(&upload_uuid);
let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(staging.join("xl.meta")).expect("upload metadata"))
.expect("decode upload metadata");
let versions = metadata
.get_file_info_versions(bucket, "protected__XLDIR__", true)
.expect("upload file info");
let data_dir = versions
.versions
.first()
.expect("upload version")
.data_dir
.expect("staging data directory");
assert!(
staging.join(data_dir.to_string()).join("part.2").is_file(),
"unselected part exists before completion"
);
}
assert_eq!(
store
.get_multipart_info(bucket, "protected/", &upload.upload_id, &opts)
.await
.expect("upload info")
.object,
"protected/"
);
assert_eq!(
store
.list_object_parts(bucket, "protected/", &upload.upload_id, None, 10, &opts)
.await
.expect("list parts")
.object,
"protected/"
);
let uploads = store
.list_multipart_uploads(bucket, "protected/", None, None, None, 10)
.await
.expect("list logical prefix");
assert_eq!(uploads.uploads.len(), 1);
assert_eq!(uploads.uploads[0].object, "protected/");
assert_eq!(crate::runtime::sources::upload_uuid_suffix(&uploads.uploads[0].upload_id), upload_uuid,);
let completed = store
.clone()
.complete_multipart_upload(
bucket,
"protected/",
&upload.upload_id,
vec![CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&opts,
)
.await
.expect("complete directory upload");
assert_eq!(completed.name, "protected/");
for (key, expected) in [
("protected", b"private original".as_slice()),
("protected/", b"directory payload".as_slice()),
] {
let mut reader = store
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read distinct object");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("read body");
assert_eq!(bytes, expected);
}
let preserved = store
.get_object_info(bucket, "protected", &ObjectOptions::default())
.await
.expect("bare metadata");
assert_eq!(preserved.version_id, original.version_id);
assert_eq!(preserved.etag, original.etag);
assert_eq!(preserved.user_tags.as_str(), "classification=secret");
for dir in &dirs {
assert!(dir.path().join(bucket).join("protected/xl.meta").is_file());
assert!(dir.path().join(bucket).join("protected__XLDIR__/xl.meta").is_file());
let data = dir
.path()
.join(bucket)
.join("protected__XLDIR__")
.join(completed.data_dir.expect("multipart data directory").to_string());
assert!(data.join("part.1").is_file());
assert!(!data.join("part.2").exists(), "unselected part must never be published");
assert!(
!dir.path()
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_sha)
.join(&upload_uuid)
.exists(),
"original staging must be cleaned"
);
}
let abandoned = store
.new_multipart_upload(bucket, "protected/", &opts)
.await
.expect("create abort control");
store
.abort_multipart_upload(bucket, "protected/", &abandoned.upload_id, &opts)
.await
.expect("abort encoded upload");
assert!(
store
.get_multipart_info(bucket, "protected/", &abandoned.upload_id, &opts)
.await
.is_err()
);
assert!(
store
.get_object_info(bucket, "protected/", &ObjectOptions::default())
.await
.is_ok()
);
}
}
#[cfg(not(windows))]
#[tokio::test]
async fn new_control_character_keys_are_rejected_without_trapping_legacy_objects() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::object::ObjectIO as _;
use tokio::io::AsyncReadExt;
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "xml-key-admission";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
let key = format!("bad-{}", char::from(byte));
assert!(
store
.put_object(bucket, &key, &mut PutObjReader::from_vec(b"body".to_vec()), &ObjectOptions::default())
.await
.is_err()
);
assert!(
store
.new_multipart_upload(bucket, &key, &ObjectOptions::default())
.await
.is_err()
);
for dir in &dirs {
assert!(!dir.path().join(bucket).join(&key).exists());
}
}
let legacy_key = "legacy-\u{1}";
store.pools[0]
.get_disks_by_key(legacy_key)
.put_object(
bucket,
legacy_key,
&mut PutObjReader::from_vec(b"legacy payload".to_vec()),
&ObjectOptions {
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("seed a pre-upgrade object below new-key admission");
let mut reader = store
.get_object_reader(bucket, legacy_key, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("legacy GET stays usable");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("legacy bytes");
assert_eq!(bytes, b"legacy payload");
drop(reader);
store
.delete_object(bucket, legacy_key, ObjectOptions::default())
.await
.expect("legacy DELETE stays usable");
assert!(
store
.get_object_info(bucket, legacy_key, &ObjectOptions::default())
.await
.is_err()
);
}
fn mp(object: &str, upload_id: &str) -> MultipartInfo {
MultipartInfo {
bucket: "bucket".to_string(),
File diff suppressed because it is too large Load Diff
+336 -1
View File
@@ -85,6 +85,230 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
}
impl ECStore {
/// Prepare an inert user-object reservation on the currently admitted disk.
/// An unclaimed token expires and does not hold the object's metadata lock.
#[cfg(test)]
pub async fn reserve_local_object_metadata(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None, None)
.await
}
/// Prepare before the caller acquires bucket lifecycle or object locks.
/// The token holds the bucket publication fence through physical completion.
pub async fn reserve_local_object_metadata_at_incarnation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_at_incarnation_with_ttl(disk_ref, bucket, object, incarnation, timeout, None)
.await
}
/// Prepare a short-lived token for peers that may abandon the response.
pub async fn reserve_local_object_metadata_short(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_at_incarnation_with_ttl(
disk_ref,
bucket,
object,
incarnation,
timeout,
Some(std::time::Duration::from_secs(2)),
)
.await
}
async fn reserve_local_object_metadata_at_incarnation_with_ttl(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
ttl: Option<std::time::Duration>,
) -> DiskResult<Uuid> {
if self.ctx.lock_manager().is_disabled()
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|| is_meta_bucketname(bucket)
|| incarnation.is_nil()
|| timeout.is_zero()
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
{
return Err(DiskError::InvalidPath);
}
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
let mut cached = slot.owner.lock().await;
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
Some(owner) => owner,
None => {
// Bucket deletion takes publication before lifecycle and metadata.
let lock = self
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.map_err(|error| DiskError::other(error.to_string()))?;
let publication = lock
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.map_err(|error| DiskError::other(error.to_string()))?;
// Older peers may acquire lifecycle without publication.
let lifecycle = self
.acquire_bucket_incarnation_fence(bucket, incarnation)
.await
.map_err(|error| DiskError::other(error.to_string()))?;
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
if !owner.is_reusable() {
return Err(DiskError::DiskOngoingReq);
}
*cached = Arc::downgrade(&owner);
owner
}
};
drop(cached);
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner), ttl)
.await
})
.await
.map_err(|_| DiskError::Timeout)?
}
async fn reserve_local_object_metadata_with_owner(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
timeout: std::time::Duration,
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
ttl: Option<std::time::Duration>,
) -> DiskResult<Uuid> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
return Err(DiskError::InvalidPath);
}
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
match ttl {
Some(ttl) => {
wrapper
.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
.await
}
None => wrapper.prepare_object_metadata_with_owner(bucket, object, owner).await,
}
})
.await
.map_err(|_| DiskError::Timeout)?
}
/// Refresh an inert token on its original disk and object.
pub async fn renew_local_object_metadata_reservation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
token: Uuid,
) -> DiskResult<()> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
return Err(DiskError::InvalidPath);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
wrapper.renew_object_metadata(bucket, object, token).await
}
/// Atomically enqueue a prepared token on its original disk and object.
pub async fn activate_local_object_metadata_reservation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
token: Uuid,
) -> DiskResult<()> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
return Err(DiskError::InvalidPath);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
wrapper.activate_object_metadata(bucket, object, token).await
}
/// Cancel an unclaimed token on the currently admitted disk.
/// A token already claimed by a physical mutation cannot be cancelled.
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
Ok(wrapper.cancel_object_metadata_reservation(token))
}
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
/// The token is bound to the admitted disk instance and destination. A claimed
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
/// The caller must retain the bucket lifecycle fence until the physical tail
/// finishes; the reservation does not identify a bucket incarnation.
pub async fn rename_local_data_reserved(
&self,
disk_ref: &str,
source: (&str, &str),
fi: &FileInfo,
destination: (&str, &str),
token: Uuid,
) -> DiskResult<RenameDataResp> {
if token.is_nil()
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|| is_meta_bucketname(destination.0)
|| rustfs_utils::http::metadata_compat::contains_key_str(
&fi.metadata,
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
)
{
return Err(DiskError::InvalidPath);
}
rename_local_data_with_ctx(
&self.ctx,
disk_ref,
source,
fi,
destination,
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
)
.await
}
/// Resolve a quota lease against this request's captured storage instance.
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
@@ -270,6 +494,7 @@ impl ECStore {
scanner_publication_lease_token: scanner_token,
external_guard,
namespace_owner: None,
..Default::default()
},
)
.await
@@ -351,7 +576,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
Ok(())
}
async fn admit_local_disk(
pub(crate) async fn admit_local_disk(
ctx: &Arc<InstanceContext>,
disk: &DiskStore,
disk_id: Option<Uuid>,
@@ -783,6 +1008,116 @@ mod tests {
.expect("seeded metadata bytes")
}
#[tokio::test]
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
let ctx = Arc::new(InstanceContext::new());
let dir = tempfile::tempdir().expect("disk root");
let id = Uuid::new_v4();
let disk = target_disk(&ctx, dir.path(), id).await;
let disk_ref = disk.endpoint().to_string();
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let version = Uuid::new_v4();
let fi = target_file_info("object", version, b"reserved");
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
.await
.expect("user bucket");
assert!(matches!(
store
.reserve_local_object_metadata(
&disk_ref,
"target-bucket/../.rustfs.sys",
"object",
std::time::Duration::from_secs(5),
)
.await,
Err(DiskError::InvalidPath)
));
assert!(matches!(
store
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
.await,
Err(DiskError::InvalidPath)
));
let token = store
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
.await
.expect("reserve current disk");
assert!(matches!(
store
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
.await,
Err(DiskError::InvalidPath)
));
store
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
.await
.expect("activate token");
store
.rename_local_data_reserved(
&disk_ref,
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
&fi,
("target-bucket", "object"),
token,
)
.await
.expect("reserved publication");
assert!(
!store
.cancel_local_object_metadata_reservation(&disk_ref, token)
.await
.expect("consumed token")
);
let read = disk
.read_version(
"target-bucket",
"target-bucket",
"object",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read reserved publication");
assert_eq!(read.data, fi.data);
let stale = store
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
.await
.expect("reserve before disk replacement");
let replacement = target_disk(&ctx, dir.path(), id).await;
assert!(matches!(
store
.rename_local_data_reserved(
&disk_ref,
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
&fi,
("target-bucket", "object"),
stale,
)
.await,
Err(DiskError::DiskOngoingReq)
));
let after = replacement
.read_version(
"target-bucket",
"target-bucket",
"object",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("replacement retains publication");
assert_eq!(after.data, fi.data);
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
}
#[tokio::test]
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
for warm in [false, true] {
+18
View File
@@ -604,6 +604,24 @@ impl ECStore {
Ok(idx)
}
/// A speculative hint only: callers must repeat the locked selection before committing.
pub(super) async fn get_pool_idx_for_reservation_hint(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
let opts = ObjectOptions {
no_lock: true,
metadata_cache_safe: true,
skip_decommissioned: true,
skip_rebalancing: true,
..Default::default()
};
match self.get_pool_idx_existing_with_opts(bucket, object, &opts).await {
Ok(idx) => Ok(idx),
Err(err) if is_err_object_not_found(&err) => {
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
}
Err(err) => Err(err),
}
}
pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await {
Ok(res) => res,
@@ -20,230 +20,14 @@ use std::sync::Arc;
use std::time::Duration;
use storage_api::contract_compat::ECStore;
use storage_api::metadata_lock::{
BucketOperations, CompletePart, DeleteBucketOptions, Error, HashReader, MakeBucketOptions, MultipartOperations,
NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause,
init_bucket_metadata_sys, isolated_store_over_temp_disks,
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
isolated_store_over_temp_disks,
};
use tokio::io::AsyncReadExt;
use tokio::time::timeout;
use uuid::Uuid;
struct ObservedPutBody {
inner: tokio::io::DuplexStream,
polled: Arc<tokio::sync::Notify>,
}
impl tokio::io::AsyncRead for ObservedPutBody {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
self.polled.notify_one();
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
#[tokio::test]
async fn destructive_put_takes_bucket_locks_only_after_its_body_is_staged() {
use tokio::io::AsyncWriteExt;
for (object, version_id) in [("object", None), ("directory/", Some(Uuid::nil().to_string()))] {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "deferred-put-locks";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let encoded_object = if object.ends_with('/') { "directory__XLDIR__" } else { object };
let barrier = PutObjectCommitBarrier::install(bucket, encoded_object, PutObjectCommitPause::BeforeQuotaRename);
let (mut sender, inner) = tokio::io::duplex(16);
let polled = Arc::new(tokio::sync::Notify::new());
let body = ObservedPutBody {
inner,
polled: polled.clone(),
};
let put_store = store.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
put_store
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
version_id,
..Default::default()
},
)
.await
});
timeout(Duration::from_secs(5), polled.notified())
.await
.expect("PUT must be waiting for client data");
let lock = store
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
.await
.expect("metadata lock");
drop(
timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5)))
.await
.expect("body must not block metadata writers")
.expect("metadata write guard"),
);
drop(
timeout(Duration::from_secs(2), store.lock_bucket_for_recursive_delete(bucket))
.await
.expect("body must not retain lifecycle guard")
.expect("lifecycle writer"),
);
sender.write_all(b"body").await.expect("finish body");
sender.shutdown().await.expect("EOF");
timeout(Duration::from_secs(5), barrier.wait_until_paused())
.await
.expect("staged PUT reaches commit");
let writer = lock.get_write_lock(Duration::from_secs(5));
tokio::pin!(writer);
assert!(futures::poll!(&mut writer).is_pending(), "commit must fence metadata updates");
let lifecycle = store.lock_bucket_for_recursive_delete(bucket);
tokio::pin!(lifecycle);
assert!(futures::poll!(&mut lifecycle).is_pending(), "commit must fence bucket deletion");
barrier.release();
let release_metadata = async {
drop(writer.await.expect("metadata writer"));
};
let release_lifecycle = async {
drop(lifecycle.await.expect("lifecycle writer"));
};
timeout(Duration::from_secs(5), async {
let (put, (), ()) = tokio::join!(put, release_metadata, release_lifecycle);
put.expect("owner task").expect("commit");
})
.await
.expect("PUT and queued writers finish");
}
}
#[tokio::test]
async fn staged_put_cannot_publish_into_a_recreated_bucket() {
use tokio::io::AsyncWriteExt;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "deferred-put-generation";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let (mut sender, inner) = tokio::io::duplex(16);
let polled = Arc::new(tokio::sync::Notify::new());
let body = ObservedPutBody {
inner,
polled: polled.clone(),
};
let put_store = store.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
put_store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
});
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
store
.delete_bucket(
bucket,
&DeleteBucketOptions {
force: true,
..Default::default()
},
)
.await
.expect("remove old bucket while PUT stalls");
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
sender.write_all(b"body").await.expect("finish body");
sender.shutdown().await.expect("EOF");
let error = timeout(Duration::from_secs(5), put)
.await
.expect("commit gate")
.expect("owner")
.expect_err("old request generation must not publish");
assert!(
error.to_string().contains("generation") || matches!(error, Error::BucketNotFound(..)),
"{error:?}"
);
assert!(
store
.get_object_info(bucket, "object", &ObjectOptions::default())
.await
.is_err()
);
}
#[tokio::test]
async fn staged_put_rechecks_current_object_lock_at_commit() {
use tokio::io::AsyncWriteExt;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "deferred-put-retention";
store
.make_bucket(
bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("locked bucket");
let mut old = PutObjReader::from_vec(b"old".to_vec());
store
.put_object(bucket, "object", &mut old, &ObjectOptions::default())
.await
.expect("unlocked predecessor");
let (mut sender, inner) = tokio::io::duplex(16);
let polled = Arc::new(tokio::sync::Notify::new());
let body = ObservedPutBody {
inner,
polled: polled.clone(),
};
let put_store = store.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
put_store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
});
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
let mut protected = PutObjReader::from_vec(b"protected".to_vec());
store
.put_object(
bucket,
"object",
&mut protected,
&ObjectOptions {
user_defined: std::collections::HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
..Default::default()
},
)
.await
.expect("install a protected predecessor while the client stalls");
sender.write_all(b"body").await.expect("finish stale body");
sender.shutdown().await.expect("EOF");
let error = timeout(Duration::from_secs(5), put)
.await
.expect("commit gate")
.expect("task")
.expect_err("pre-body state cannot admit an overwrite of the newly protected version");
assert!(matches!(error, Error::PrefixAccessDenied(..)), "{error:?}");
let info = store
.get_object_info(bucket, "object", &ObjectOptions::default())
.await
.expect("protected object remains");
assert_eq!(info.size, 9);
}
#[tokio::test]
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
replica_write_with_waiting_metadata_writer(false).await;
+1 -3
View File
@@ -9,14 +9,12 @@ use rustfs_storage_api as storage_contracts;
#[cfg(feature = "test-util")]
pub(crate) mod metadata_lock {
pub(crate) use super::storage_contracts::{
BucketOperations, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO,
ObjectOperations,
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
};
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
pub(crate) use rustfs_ecstore::api::rio::HashReader;
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
}
-28
View File
@@ -3930,34 +3930,6 @@ mod test {
assert!(versions_with_free.versions.iter().all(|version| version.num_versions == 2));
}
#[test]
fn add_version_rejects_mod_times_that_cannot_round_trip() {
for nanos in [0, i128::from(i64::MAX) + 1, i128::from(i64::MIN) - 1, 1_i128 << 64] {
for deleted in [false, true] {
let mut meta = FileMeta::new();
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
info.deleted = deleted;
info.mod_time = Some(OffsetDateTime::from_unix_timestamp_nanos(nanos).expect("time within supported date range"));
assert!(
meta.add_version(info).is_err(),
"unrepresentable time {nanos} must be rejected before publication"
);
}
}
for nanos in [-1, 1, i64::MIN, i64::MAX] {
let mut meta = FileMeta::new();
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
let time = OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).expect("valid codec time");
info.mod_time = Some(time);
meta.add_version(info).expect("representable times remain supported");
let reopened = FileMeta::load(&meta.marshal_msg().expect("serialize version")).expect("reopen version");
let versions = reopened
.get_file_info_versions("bucket", "object", false)
.expect("read version history");
assert_eq!(versions.versions[0].mod_time, Some(time));
}
}
#[test]
fn test_data_integrity_validation() {
// Test data integrity checks

Some files were not shown because too many files have changed in this diff Show More