Compare commits

..

8 Commits

Author SHA1 Message Date
hector d52d720d17 Merge branch 'main' into ci/validation-sweep-boto3
Keep main's validation sweep workflow where both sides edited the
same steps. The Python 3.12 venv install from
validation_ilm/requirements.txt is already on main and remains in
the sweep job, with the scheduled nightly pin, chain envelope read,
ssh -n, AUTO_TESTING_SHA, and artifact warn/run path.

Co-authored-by: hector <majinghe@users.noreply.github.com>
2026-11-02 12:39:52 +00:00
Hauser Zhi 4572bccea1 Merge branch 'main' into ci/validation-sweep-boto3 2026-11-02 18:07:28 +08:00
hector ccc25bce07 ci(validation): install the validation Python client requirements
The pool runner dies with 'boto3 is required for pool validation'
before its assertions: the managed interpreter alone is not enough.
Mirror the chain validation job exactly - build a venv from
validation_ilm/requirements.txt and put its bin on PATH so every
module runner (pool/ilm/replication) gets its client stack.
2026-11-02 16:38:53 +08:00
hector 6199e30ffc Merge branch 'main' into ci/validation-sweep-schedule-fix 2026-11-02 15:01:41 +08:00
hector 835b84c57a ci(validation): install sweep Python client dependencies
Scheduled sweeps failed pool.status before any assertion because
pool_behavior.py exits when boto3 is missing. Install the auto-testing
validation requirements into a per-run virtualenv and put it on PATH,
matching the functional chain, so case scripts can import the client.

Co-authored-by: hector <majinghe@users.noreply.github.com>
2026-11-02 06:56:18 +00:00
hector 14a2fd9569 ci(validation): sweep harness fixes from the first live run
- ssh -n in the inter-case cleanup loop: plain ssh consumed the case
  list from the loop's stdin, so the first dispatch executed only one
  of the twelve selected cases.
- Provide the managed Python 3.12 runtime for pool/ilm/replication
  cases, mirroring the chain validation job (pool.status reported
  INFRA_ERROR after its assertions passed; the missing runtime is the
  prime suspect).
- Record the auto-testing sha in the report step environment.
- Retain the last case's evidence tree in the sweep artifact so gate
  verdicts like INFRA_ERROR can be diagnosed from the run itself.
2026-11-02 09:27:20 +08:00
hector bf201caf34 ci(validation): read the resolver chain envelope correctly
CHAIN_OUTPUT is the chain envelope {schema, run_id, attempt,
workflow_sha, testing_sha, candidate}; the manifest lives under
candidate.manifest and the testing sha at the top level.
2026-11-02 09:17:09 +08:00
hector 4d81330479 ci(validation): resolve the newest nightly build on scheduled sweeps
The first scheduled sweep failed at candidate resolution: a schedule
carries no workflow inputs and resolve_functional_candidate.py only
derives a build from a workflow_run payload, so the empty
BUILD_RUN_ID crashed int(). Look up the newest successful
nightly-gnu run explicitly when the inputs are empty and feed it to
the resolver. Also downgrade the evidence artifact to
if-no-files-found: warn so an early infra failure does not cascade
into a second red step.
2026-11-02 09:09:20 +08:00
337 changed files with 4557 additions and 43590 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=a3e166a140f3150218b4a857fedb8749940fad0c8b053d0e3c1b957321bf5beb
sha256-linux=dce4a8eb634a27b4589bcf6d2ebf248ede97aa59701acc8696391d2812a3177d
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
+1 -1
View File
@@ -1 +1 @@
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
+2 -2
View File
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
[profile.e2e-smoke]
default-filter = """
package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/)
| test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Nightly and manual-dispatch lane owned by
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+3 -3
View File
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
| File | Description | Usage |
| :--- | :--- | :--- |
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
### Docker Compose Configurations
-2
View File
@@ -32,8 +32,6 @@ updates:
- "overtrue"
- "majinghe"
ignore:
- dependency-name: "pulsar"
versions: [ "6.9.x" ]
- dependency-name: "object_store"
versions: [ "0.13.x" ]
- dependency-name: "libunftp"
+5 -2
View File
@@ -769,10 +769,12 @@
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"workflow_dispatch",
"schedule"
],
@@ -847,11 +849,12 @@
"workflow": ".github/workflows/e2e-upgrade.yml",
"name": "Upgrade Compatibility",
"roles": [
"pr",
"periodic",
"manual",
"release"
"manual"
],
"events": [
"pull_request",
"push",
"schedule",
"workflow_dispatch"
+1 -60
View File
@@ -113,7 +113,7 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
@@ -497,63 +497,6 @@ jobs:
- name: Check offline enrollment E2E root boundary
run: ./scripts/check_offline_enrollment_e2e.sh
fips-provider:
name: FIPS provider boundaries
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
mode: [default, fips]
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
CARGO_BUILD_JOBS: "2"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-fips-provider
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install FIPS build tools
if: matrix.mode == 'fips'
run: sudo apt-get install -y cmake golang clang libclang-dev
- name: Verify provider installation
shell: bash
run: |
args=()
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
args+=(--features fips)
fi
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
fi
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
fi
for test_name in "${tests[@]}"; do
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
done
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
# Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1454,7 +1397,6 @@ jobs:
- workspace-shard-evidence
- core-contract
- connect-feature-boundaries
- fips-provider
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
@@ -1482,7 +1424,6 @@ jobs:
needs:
- classify-changes
- connect-feature-boundaries
- fips-provider
- required-checks
- typos
- quick-checks
+28 -2
View File
@@ -17,8 +17,8 @@
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
# and manual dispatches execute the same fail-closed suite.
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".config/e2e-distributed-selection.txt"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
+18 -7
View File
@@ -12,19 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Single-node upgrade and rollback lane.
#
# The weekly Monday schedule, version tags, and workflow_dispatch run this
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
name: Upgrade Compatibility
on:
pull_request:
paths:
- ".github/workflows/e2e-upgrade.yml"
- "crates/e2e_test/src/common.rs"
- "crates/e2e_test/src/distributed/**"
- "crates/e2e_test/src/fake_s3_target/**"
- "crates/e2e_test/src/lib.rs"
- "crates/e2e_test/src/replication_extension_test.rs"
- "crates/e2e_test/src/rpc_fault_proxy.rs"
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/kms/**"
- "crates/storage-api/**"
- "rustfs/**"
- "Cargo.lock"
push:
tags:
- "[0-9]*.[0-9]*.[0-9]*"
schedule:
# 03:17 UTC every Monday.
- cron: "17 3 * * 1"
workflow_dispatch:
@@ -33,7 +44,7 @@ permissions:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
+7 -279
View File
@@ -87,206 +87,18 @@ jobs:
install-build-packaging-tools: 'false'
install-test-tools: 'false'
- name: Download static console assets
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
# Packaging tools are off on this lane, so setup does not install unzip.
if ! command -v unzip >/dev/null 2>&1; then
sudo apt-get update -qq
sudo apt-get install -y -qq unzip
fi
if ! command -v unzip >/dev/null 2>&1; then
echo "unzip is required to extract console assets" >&2
exit 1
fi
mkdir -p ./rustfs/static
verify_sha256() {
local expected="$1"
local file="$2"
local actual=""
local actual_line=""
if command -v sha256sum >/dev/null 2>&1; then
actual_line=$(sha256sum -- "$file") || return 1
elif command -v shasum >/dev/null 2>&1; then
actual_line=$(shasum -a 256 -- "$file") || return 1
else
echo "No SHA-256 verification tool found" >&2
return 1
fi
actual="${actual_line%%[[:space:]]*}"
if [ "$actual" = "$expected" ]; then
echo "SHA256 verified OK: $actual"
return 0
else
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
return 1
fi
}
download_console_assets() {
local curl_bin="curl"
local python_bin="python3"
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
local console_json="console-release.json"
local console_url
local console_sha256
local curl_auth_args=()
console_tag=""
chmod +w ./rustfs/static/LICENSE || true
if ! command -v "$python_bin" >/dev/null 2>&1; then
python_bin="python"
fi
if ! command -v "$python_bin" >/dev/null 2>&1; then
echo "No Python interpreter found for release metadata parsing" >&2
return 1
fi
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
fi
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
import json
import re
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
release = json.load(handle)
tag = release.get("tag_name", "") or "unknown"
for asset in release.get("assets", []):
name = asset.get("name", "")
digest = asset.get("digest", "")
url = asset.get("browser_download_url", "")
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
sha256 = digest.split(":", 1)[1]
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
break
else:
raise SystemExit("no console zip asset with sha256 digest found")
PY
) || return 1
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
echo "Console release metadata is missing URL or SHA-256 digest" >&2
return 1
fi
echo "Console release: ${console_tag}"
echo "Downloading console asset: ${console_url}"
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
verify_sha256 "$console_sha256" console.zip || return 2
unzip -o console.zip -d ./rustfs/static || return 2
}
if download_console_assets; then
rm -f console.zip console-release.json
else
status=$?
rm -f console.zip console-release.json
if [[ "$status" -eq 2 ]]; then
echo "Console asset integrity verification failed" >&2
fi
echo "Failed to download verified console assets" >&2
exit 1
fi
if [[ ! -s ./rustfs/static/index.html ]]; then
echo "Console asset archive is missing static/index.html" >&2
exit 1
fi
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
- name: Build RustFS
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
shell: bash
run: |
# Force rebuild by touching build.rs so a restored cache cannot
# reuse a rustfs artifact compiled before these console assets existed.
touch rustfs/build.rs
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
- name: Verify glibc symbol baseline
run: |
python3 scripts/release/check_glibc.py 2.31 \
target/x86_64-unknown-linux-gnu/release/rustfs
target/x86_64-unknown-linux-gnu/release/rustfs \
target/x86_64-unknown-linux-gnu/release/rustfs-cli
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
- name: Verify console
shell: bash
run: |
set -euo pipefail
find_free_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
data_dir="$(mktemp -d)"
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
server_pid=""
trap '
if [[ -n "${server_pid:-}" ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
fi
rm -rf "$data_dir"
' EXIT
if [[ ! -x "$binary" ]]; then
echo "rustfs binary not found: ${binary}" >&2
exit 1
fi
api_port="$(find_free_port)"
console_port="$(find_free_port)"
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
"$binary" server \
--address "127.0.0.1:${api_port}" \
--console-enable \
--console-address "127.0.0.1:${console_port}" \
--access-key console-smoke \
--secret-key console-smoke-secret \
"$data_dir" >"$log_file" 2>&1 &
server_pid=$!
response=""
for _ in {1..40}; do
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
if [[ "$response" == 200\ text/html* ]]; then
exit 0
fi
sleep 0.25
done
echo "Console endpoint did not return 200 text/html: $response" >&2
cat "$log_file"
exit 1
- name: Build DEB package
id: deb
shell: bash
@@ -534,12 +346,12 @@ jobs:
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
# engineers can download and install the nightly directly. The branch is
# a single-commit orphan rewritten on every build, which keeps the repo
# small while the latest files stay reachable at stable raw URLs.
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
# The dispatch below runs only after this upload and the immutable R2
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
# which still starts from this workflow's workflow_run completion.
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
- name: Publish packages to auto-testing release assets
env:
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -548,90 +360,6 @@ jobs:
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
run: bash scripts/release/publish_nightly_assets.sh
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
# or fine-grained Contents write (the permission GitHub checks for
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
# step, and therefore this job, so the scheduled failure alert fires.
# GitHub allows at most 10 top-level client_payload properties, so the
# schema 2 candidate object is nested under `candidate` and the release
# fields sit beside it.
- name: Dispatch nightly candidate to auto-testing
if: ${{ steps.publish.outputs.candidate_file != '' }}
env:
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
SOURCE_REPO: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
: "${DEB_DATE:?DEB_DATE is required}"
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
--arg event_type "nightly-candidate-ready" \
--arg source_repo "${SOURCE_REPO}" \
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
--arg channel "nightly" \
--arg deb_date "${DEB_DATE}" \
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
'{
event_type: $event_type,
source_repo: $source_repo,
latest_alias_url: $latest_alias_url,
auto_testing_release: $auto_testing_release,
channel: $channel,
deb_date: $deb_date,
nightly_workflow: $nightly_workflow,
candidate: $candidate[0]
}' > "${payload}"
if ! jq -e '
.event_type == "nightly-candidate-ready"
and .source_repo == "rustfs/rustfs"
and .channel == "nightly"
and .deb_date == env.DEB_DATE
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
and .candidate.schema == 2
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
' "${payload}" >/dev/null; then
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
exit 1
fi
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile client_payload "${payload}" \
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-o "${response}" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "Content-Type: application/json" \
--data-binary @"${body}" \
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
exit 1
}
if [[ "${status}" != "204" ]]; then
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
if [[ -s "${response}" ]]; then
cat "${response}" >&2
fi
exit 1
fi
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
-1
View File
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
### Security
Generated
+150 -272
View File
@@ -40,9 +40,9 @@ dependencies = [
[[package]]
name = "aes"
version = "0.9.4"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1"
checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32"
dependencies = [
"cipher 0.5.2",
"cpubits",
@@ -57,7 +57,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
dependencies = [
"aead",
"aes 0.9.4",
"aes 0.9.3",
"cipher 0.5.2",
"ctr",
"ctutils",
@@ -370,17 +370,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df"
dependencies = [
"arrow-arith",
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-csv",
"arrow-data 59.3.2",
"arrow-ipc 59.3.2",
"arrow-data",
"arrow-ipc",
"arrow-json",
"arrow-ord",
"arrow-row",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-schema",
"arrow-select",
"arrow-string",
]
@@ -390,10 +390,10 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"num-traits",
]
@@ -405,9 +405,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43"
dependencies = [
"ahash",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"chrono-tz",
"half",
@@ -417,24 +417,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "arrow-array"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"
dependencies = [
"ahash",
"arrow-buffer 60.1.0",
"arrow-data 60.1.0",
"arrow-schema 60.1.0",
"chrono",
"half",
"hashbrown 0.17.1",
"num-complex",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "59.3.2"
@@ -447,30 +429,18 @@ dependencies = [
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"
dependencies = [
"bytes",
"half",
"num-bigint 0.5.1",
"num-traits",
]
[[package]]
name = "arrow-cast"
version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ord",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-schema",
"arrow-select",
"atoi 2.0.0",
"base64 0.23.1",
"chrono",
@@ -481,26 +451,15 @@ dependencies = [
"ryu",
]
[[package]]
name = "arrow-cmp"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"
dependencies = [
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-schema 60.1.0",
]
[[package]]
name = "arrow-csv"
version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698"
dependencies = [
"arrow-array 59.3.2",
"arrow-array",
"arrow-cast",
"arrow-schema 59.3.2",
"arrow-schema",
"chrono",
"csv",
"csv-core",
@@ -513,21 +472,8 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba"
dependencies = [
"arrow-buffer 59.3.2",
"arrow-schema 59.3.2",
"half",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-data"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"
dependencies = [
"arrow-buffer 60.1.0",
"arrow-schema 60.1.0",
"arrow-buffer",
"arrow-schema",
"half",
"num-integer",
"num-traits",
@@ -539,43 +485,28 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"flatbuffers",
"lz4_flex",
"zstd 0.13.4",
]
[[package]]
name = "arrow-ipc"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"
dependencies = [
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-data 60.1.0",
"arrow-schema 60.1.0",
"arrow-select 60.1.0",
"flatbuffers",
"zstd 0.14.3",
]
[[package]]
name = "arrow-json"
version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-ord",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-schema",
"arrow-select",
"chrono",
"half",
"indexmap 2.14.2",
@@ -595,11 +526,11 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
]
[[package]]
@@ -608,10 +539,10 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"half",
]
@@ -625,12 +556,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "arrow-schema"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"
[[package]]
name = "arrow-select"
version = "59.3.2"
@@ -638,25 +563,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4"
dependencies = [
"ahash",
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"num-traits",
]
[[package]]
name = "arrow-select"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"
dependencies = [
"ahash",
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-cmp",
"arrow-data 60.1.0",
"arrow-schema 60.1.0",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"num-traits",
]
@@ -666,11 +576,11 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"memchr",
"num-traits",
"regex",
@@ -759,9 +669,9 @@ dependencies = [
[[package]]
name = "async-compression"
version = "0.4.52"
version = "0.4.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459"
checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970"
dependencies = [
"compression-codecs",
"compression-core",
@@ -772,9 +682,9 @@ dependencies = [
[[package]]
name = "async-executor"
version = "1.14.1"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37"
checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a"
dependencies = [
"async-task",
"concurrent-queue",
@@ -800,9 +710,9 @@ dependencies = [
[[package]]
name = "async-lock"
version = "3.5.0"
version = "3.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f"
checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311"
dependencies = [
"event-listener",
"event-listener-strategy",
@@ -2083,9 +1993,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.11.0"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b"
checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2341,9 +2251,9 @@ dependencies = [
[[package]]
name = "compression-codecs"
version = "0.4.47"
version = "0.4.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43"
checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
dependencies = [
"brotli 9.1.1",
"bzip2",
@@ -2580,18 +2490,18 @@ dependencies = [
[[package]]
name = "crc32c"
version = "0.6.9"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f132a176423ee852e8f0a35d43088d1850966b50745be818a88b76f64b1c6cb"
checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
dependencies = [
"rustc_version",
]
[[package]]
name = "crc32fast"
version = "1.5.4"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04"
checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
dependencies = [
"cfg-if",
]
@@ -2766,9 +2676,9 @@ dependencies = [
[[package]]
name = "ctutils"
version = "0.4.8"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f"
checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a"
dependencies = [
"cmov",
"subtle",
@@ -2981,7 +2891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"chrono",
"datafusion-catalog",
@@ -3015,7 +2925,7 @@ dependencies = [
"log",
"object_store",
"parking_lot 0.12.5",
"parquet 59.3.2",
"parquet",
"sqlparser",
"tempfile",
"tokio",
@@ -3079,8 +2989,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687"
dependencies = [
"arrow",
"arrow-ipc 59.3.2",
"arrow-schema 59.3.2",
"arrow-ipc",
"arrow-schema",
"chrono",
"foldhash 0.2.0",
"half",
@@ -3091,7 +3001,7 @@ dependencies = [
"log",
"num-traits",
"object_store",
"parquet 59.3.2",
"parquet",
"recursive",
"sqlparser",
"tokio",
@@ -3147,7 +3057,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb"
dependencies = [
"arrow",
"arrow-ipc 59.3.2",
"arrow-ipc",
"async-trait",
"bytes",
"datafusion-common",
@@ -3217,7 +3127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"bytes",
"datafusion-common",
@@ -3238,7 +3148,7 @@ dependencies = [
"log",
"object_store",
"parking_lot 0.12.5",
"parquet 59.3.2",
"parquet",
"tokio",
]
@@ -3255,7 +3165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837"
dependencies = [
"arrow",
"arrow-buffer 59.3.2",
"arrow-buffer",
"async-trait",
"bytes",
"dashmap",
@@ -3281,7 +3191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"chrono",
"datafusion-common",
@@ -3317,7 +3227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771"
dependencies = [
"arrow",
"arrow-buffer 59.3.2",
"arrow-buffer",
"base64 0.23.1",
"chrono",
"chrono-tz",
@@ -3477,7 +3387,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
@@ -3552,10 +3462,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97"
dependencies = [
"arrow",
"arrow-data 59.3.2",
"arrow-ipc 59.3.2",
"arrow-data",
"arrow-ipc",
"arrow-ord",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"bytes",
"datafusion-common",
@@ -3603,7 +3513,7 @@ version = "55.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a"
dependencies = [
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"datafusion-common",
"datafusion-execution",
@@ -3906,9 +3816,9 @@ dependencies = [
[[package]]
name = "dial9"
version = "0.5.7"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f3298a54fd82af243c9f122595762f5b0e48bd3d1ea8fe3f3eb5e95c130e881"
checksum = "11185c62245c5ff61c753ec8ec3d56341a13fe4195344ecc100ce94f780533cc"
dependencies = [
"bon",
"dial9-core",
@@ -3923,9 +3833,9 @@ dependencies = [
[[package]]
name = "dial9-core"
version = "0.5.7"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc7907b149a394404e3075f57bfbc751c7292ef115027f13f4995ff3a8b360c"
checksum = "54f1dcb5d20bdc85b9beb7c2864540526c1394c00783e9f3544feed1cfe529e8"
dependencies = [
"ahash",
"arc-swap",
@@ -3957,9 +3867,9 @@ dependencies = [
[[package]]
name = "dial9-metrique"
version = "0.5.7"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35f30e80d69214ed7c684d81afd8e52315b4acbfa5be9ebb84ccee02ebd06f44"
checksum = "56645773b4185f51a9528625a5aedcd4bbc5b5919ba68e7a6108cf076f635efc"
dependencies = [
"dial9-core",
"dial9-trace-format",
@@ -3971,9 +3881,9 @@ dependencies = [
[[package]]
name = "dial9-perf-self-profile"
version = "0.5.7"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17f09434b7623ef8b9b107062941a8e471e6044d6c946fa4af899130223b7a28"
checksum = "00c8c96d18b28ec76adde2107f7d2d5f8bb53b0670fba31bc012841703bc49ca"
dependencies = [
"blazesym",
"bon",
@@ -3990,9 +3900,9 @@ dependencies = [
[[package]]
name = "dial9-tokio-telemetry"
version = "0.5.7"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0428cba60fbcf37ab82666f3a7ef3b6d0671e90dfb6a352376813b471319f061"
checksum = "7ccc79f3683219471a8758b8ce459589167c55d72a649d27b89068a1a4cfffe9"
dependencies = [
"bon",
"bytes",
@@ -4015,9 +3925,9 @@ dependencies = [
[[package]]
name = "dial9-trace-format"
version = "0.5.4"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af753ba562fab9b99024b74234fe3a93b01c32d6474db8ae4f2c3bb562958f8f"
checksum = "44026339a4c6cbd4914c3eb85e653e741d5739f1e401695a039127ba49c19731"
dependencies = [
"ahash",
"dial9-trace-format-derive",
@@ -4027,9 +3937,9 @@ dependencies = [
[[package]]
name = "dial9-trace-format-derive"
version = "0.5.4"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e00786284c5d77b60aac60ee96797b8ac020306da8f02fecb0e86c5e82610aae"
checksum = "bbdbf5c76eb6824b0576e6891811caae93fc0bc96c23003556e3f752d069a84d"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@@ -4731,6 +4641,7 @@ version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
dependencies = [
"generic-array 0.14.7",
"rustversion",
"typenum",
]
@@ -5894,9 +5805,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jiff"
version = "0.2.39"
version = "0.2.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ac703058abac89e5c39b0b879dce2b6b14a4ba36600c7e7e3d0c60a6883ef3c"
checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383"
dependencies = [
"defmt 1.2.0",
"jiff-core",
@@ -5911,9 +5822,9 @@ dependencies = [
[[package]]
name = "jiff-core"
version = "0.1.2"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aee43309eb53bafb8c36d35d6e444769c8f8e6206a9fbf7546f5db9d7d4a0e93"
checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c"
dependencies = [
"defmt 1.2.0",
"log",
@@ -5921,9 +5832,9 @@ dependencies = [
[[package]]
name = "jiff-static"
version = "0.2.39"
version = "0.2.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dc32470c67eb7d0e9ccb74303fa925313dc04cecb1aa048beaa6f391e36a93d"
checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c"
dependencies = [
"jiff-core",
"proc-macro2",
@@ -6412,9 +6323,9 @@ dependencies = [
[[package]]
name = "lzma-rust2"
version = "0.24.0"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490750560fa7851fbbf5d3ab1a8750f09350ed361ed8aed85b1e1e40587812ee"
checksum = "f1c46f3fcd2e888afdb01a07efc071cb21300e07c68d32c3e8b01b5c8cb1fcaa"
dependencies = [
"sha2 0.11.1",
]
@@ -7680,12 +7591,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680"
dependencies = [
"ahash",
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-ipc 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ipc",
"arrow-schema",
"arrow-select",
"base64 0.23.1",
"brotli 8.0.4",
"bytes",
@@ -7707,32 +7618,6 @@ dependencies = [
"zstd 0.13.4",
]
[[package]]
name = "parquet"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"
dependencies = [
"ahash",
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-data 60.1.0",
"arrow-ipc 60.1.0",
"arrow-schema 60.1.0",
"arrow-select 60.1.0",
"base64 0.23.1",
"bytes",
"chrono",
"half",
"hashbrown 0.17.1",
"num-bigint 0.5.1",
"num-integer",
"num-traits",
"seq-macro",
"twox-hash",
"zstd 0.14.3",
]
[[package]]
name = "password-hash"
version = "0.6.1"
@@ -8039,7 +7924,7 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
dependencies = [
"aes 0.9.4",
"aes 0.9.3",
"aes-gcm",
"cbc 0.2.1",
"der 0.8.3",
@@ -9091,9 +8976,9 @@ dependencies = [
[[package]]
name = "reflink-copy"
version = "0.1.32"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad64f61357bcf16105a0f353f9783733409c8bd889bc326ad857959605ea927"
checksum = "3dd31f458a66eeda05a9e837fc5f0effbaacbe8565868ecae5b37bf0544c19f5"
dependencies = [
"cfg-if",
"libc",
@@ -9348,11 +9233,11 @@ dependencies = [
[[package]]
name = "russh"
version = "0.65.1"
version = "0.65.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94be3d86c71b9ed220a675c405e35c7a02565a009bf8da2e9cc13a700320a5c5"
checksum = "3bb77a4b259312b2abbc541bca1c13cb4e9b61c560a7fd894068c3b92e0c9dc3"
dependencies = [
"aes 0.9.4",
"aes 0.9.3",
"aws-lc-rs",
"bitflags 2.13.2",
"block-padding 0.4.2",
@@ -10090,7 +9975,6 @@ version = "1.0.2"
dependencies = [
"bytes",
"hotpath",
"metrics",
"rustfs-io-metrics",
"thiserror 2.0.21",
"tokio",
@@ -10115,9 +9999,9 @@ dependencies = [
[[package]]
name = "rustfs-kafka"
version = "1.4.1"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82dc39ba1a5a9f6b2e49decb0b54c23ebaebdfb08a899307c266694bfd269bf9"
checksum = "bd2f8b46ec4dd462c2235948d1efa2103957230c171e118c6ff8e5d1a845ab48"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -10140,9 +10024,9 @@ dependencies = [
[[package]]
name = "rustfs-kafka-async"
version = "1.4.1"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "684f5e78ea1421bbf915e4cd0515bdefd587cfa281da000e41ae4a434f90195a"
checksum = "4b3b98705f6f9f5071b8793d29935de764c31668c6e9877553b229eb67127af6"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -10277,7 +10161,6 @@ dependencies = [
"hotpath",
"parking_lot 0.13.0",
"rand 0.10.3",
"rustfs-common",
"rustfs-io-metrics",
"rustfs-utils",
"serde",
@@ -10331,18 +10214,18 @@ dependencies = [
[[package]]
name = "rustfs-mimalloc"
version = "0.7.3"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216ecfc9341b5d6e2506d47ce57c9232ae1b8020bb10264ea05286fb234286e7"
checksum = "8301a578772fc9823d68e083e42c3b414613d588a0eac594c1ca6bc0bbe0653f"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.7.3"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb16ad68f96883407a75cdb88252b43dc2ea592428f0fc54ff7885756de4b814"
checksum = "6eb4082853043cb29556b4b776b05f28c3146706f43c53869dbf1ef262e00e91"
dependencies = [
"cc",
]
@@ -10536,7 +10419,6 @@ dependencies = [
"md-5 0.11.0",
"percent-encoding",
"proptest",
"rcgen",
"regex",
"russh",
"russh-sftp",
@@ -10790,10 +10672,8 @@ version = "1.0.2"
dependencies = [
"async-trait",
"bytes",
"csv-core",
"datafusion",
"derive_builder",
"flate2",
"futures",
"hotpath",
"parking_lot 0.13.0",
@@ -11028,7 +10908,6 @@ version = "1.0.2"
dependencies = [
"async-trait",
"axum",
"futures",
"hotpath",
"http 1.5.0",
"ipnetwork",
@@ -11063,7 +10942,6 @@ dependencies = [
name = "rustfs-utils"
version = "1.0.2"
dependencies = [
"aws-lc-rs",
"base64-simd",
"blake2",
"brotli 9.1.1",
@@ -11204,14 +11082,15 @@ dependencies = [
[[package]]
name = "rustls-connector"
version = "0.23.10"
version = "0.23.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f799ef81c26fda57609b9456ae1e6750e2ca3dafc2dc5478232c0705f1458fbf"
checksum = "524994fd6957827c775abc941f41a146ce351bbd0a9b8cf632ee37f43a13c205"
dependencies = [
"futures-io",
"futures-rustls",
"log",
"rustls",
"rustls-pki-types",
"rustls-platform-verifier",
"rustls-webpki",
]
@@ -12149,7 +12028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
dependencies = [
"aead",
"aes 0.9.4",
"aes 0.9.3",
"aes-gcm",
"chacha20",
"cipher 0.5.2",
@@ -12541,18 +12420,18 @@ dependencies = [
[[package]]
name = "test-case"
version = "3.4.1"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae6e80608d2037173d7d59a09089d249e43352c182f6dde6b00d138bcb7cfdbe"
checksum = "124953e7f67cb0b2fcfb87e899e4ae5a64fb68e2e160767933cc67b646fb0042"
dependencies = [
"test-case-macros",
]
[[package]]
name = "test-case-core"
version = "3.4.1"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29b8e1f65819001927b8cdcbb5560f2428212f2e5ab5684e3c6815a4d80e72db"
checksum = "bd097615b407247e102046ba34426bb594923ab3554cf0f3cdfed050f1a5a3e8"
dependencies = [
"cfg-if",
"proc-macro2",
@@ -12562,9 +12441,9 @@ dependencies = [
[[package]]
name = "test-case-macros"
version = "3.4.1"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "802d6165a14df1598b7290a8c546df402940d3cfbe82d15c0fd35de176771693"
checksum = "f4ce35ece947bccac166e1ded639133827a953f69a48ba134d17f91108333082"
dependencies = [
"proc-macro2",
"quote",
@@ -12862,9 +12741,9 @@ dependencies = [
[[package]]
name = "toml"
version = "1.2.0+spec-1.1.0"
version = "1.1.8+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5ffcdf8f50436dbbfd675c1fe1b8e0854f42b7defea8584bdebe1aee900e561"
checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d"
dependencies = [
"indexmap 2.14.2",
"serde_core",
@@ -12886,9 +12765,9 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.25.18+spec-1.1.0"
version = "0.25.17+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ec6e78bd70db9420d8d212fa9dc97f116448a1ceff5d00eabb991d407e21396"
checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c"
dependencies = [
"indexmap 2.14.2",
"toml_datetime",
@@ -12907,9 +12786,9 @@ dependencies = [
[[package]]
name = "toml_writer"
version = "1.2.0+spec-1.1.0"
version = "1.1.3+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "009dbb7700f11c55ef83aebbe4c0e9cf1f1ac0844bd8d6f00dc38f302026e13a"
checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
[[package]]
name = "tonic"
@@ -14203,11 +14082,11 @@ dependencies = [
[[package]]
name = "zip"
version = "9.3.0"
version = "9.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaeb7f16d2605b49aef586996ded29e65642fab7259a3077e537351df6d80426"
checksum = "848dad21639597378c3be94c59fd4de683f0f6d8a5387575896e1245737d4f37"
dependencies = [
"aes 0.9.4",
"aes 0.9.3",
"bzip2",
"constant_time_eq",
"crc32fast",
@@ -14216,7 +14095,6 @@ dependencies = [
"getrandom 0.4.3",
"hmac 0.13.0",
"indexmap 2.14.2",
"libc",
"lzma-rust2",
"memchr",
"nix 0.31.3",
@@ -14260,12 +14138,12 @@ name = "zstd"
version = "0.13.4"
dependencies = [
"apache-avro",
"arrow-array 60.1.0",
"arrow-ipc 60.1.0",
"arrow-schema 60.1.0",
"arrow-array",
"arrow-ipc",
"arrow-schema",
"bytes",
"kafka-protocol",
"parquet 60.1.0",
"parquet",
"semver",
"toml",
"zstd 0.14.3",
+8 -10
View File
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-compression = { version = "0.4.51" }
async-trait = "0.1.92"
async-nats = { version = "0.51.0", default-features = false }
axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
memchr = "2.8.3"
minlz = "1.2.3"
reqwest = "0.13.5"
rustfs-kafka-async = { version = "1.4.1" }
rustfs-kafka-async = { version = "1.4.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.54.1" }
tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -191,7 +191,6 @@ quinn = { version = "0.11.12", default-features = false, features = [
# Serialization and Data Formats
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
bytes = { version = "1.12.1" }
csv-core = "0.1.13"
bytesize = "2.7.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
@@ -213,7 +212,6 @@ serde_urlencoded = "0.7.1"
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
aws-lc-rs = { version = "1.19.0", default-features = false }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0"
@@ -236,7 +234,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "0.2.39" }
jiff = { version = "0.2.38" }
time = { version = "0.3.55" }
# Database
@@ -339,7 +337,7 @@ strum = { version = "0.28.0" }
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
test-case = "3.4.0"
thiserror = "2.0.21"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
@@ -358,13 +356,13 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.3.0"
zip = "9.2.1"
zstd = "0.14.3"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.7", default-features = false }
dial9 = { version = "0.5.6", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -379,14 +377,14 @@ libunftp = { version = "0.24.0" }
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.1" }
russh = { version = "0.65.0" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.3" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.3", default-features = false }
# Snapshot testing for output format regression detection
+3 -11
View File
@@ -16,7 +16,6 @@ FROM alpine:3.24.2 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apk add --no-cache ca-certificates curl jq unzip
WORKDIR /build
@@ -70,9 +69,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
FROM alpine:3.24.2
@@ -99,18 +95,14 @@ RUN apk upgrade --no-cache && \
ca-certificates \
coreutils \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN addgroup -g 10001 -S rustfs && \
adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -132,6 +124,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+4 -13
View File
@@ -12,10 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM rust:1.99.1-trixie
ARG TARGETARCH
ARG RC_RELEASE=latest
FROM rust:1.97.1-trixie
RUN set -eux; \
export DEBIAN_FRONTEND=noninteractive; \
@@ -24,7 +21,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -40,7 +36,6 @@ COPY . .
RUN ./scripts/static.sh
RUN cargo run --bin gproto
RUN cargo build --release --locked --bin rustfs
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
RUN set -eux; \
groupadd -g 10001 rustfs; \
@@ -55,12 +50,8 @@ RUN set -eux; \
chown -R rustfs:rustfs /data /logs /app; \
chmod 0750 /data /logs
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
@@ -72,5 +63,5 @@ ENV RUSTFS_VOLUMES="/data" \
EXPOSE 9000 9001
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
+3 -11
View File
@@ -16,7 +16,6 @@ FROM ubuntu:26.04 AS build
ARG TARGETARCH
ARG RELEASE=latest
ARG RC_RELEASE=latest
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
@@ -77,9 +76,6 @@ RUN set -eux; \
chmod +x /build/rustfs; \
rm -rf rustfs.zip /build/.tmp || true
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
FROM ubuntu:26.04
ARG RELEASE=latest
@@ -103,18 +99,14 @@ RUN apt-get update && apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /build/rustfs /usr/bin/rustfs
COPY --from=build /build/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
RUN groupadd -g 10001 rustfs && \
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -139,6 +131,6 @@ VOLUME ["/data"]
USER rustfs
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["rustfs"]
+7 -19
View File
@@ -28,19 +28,17 @@ ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES=""
ARG RC_RELEASE=latest
# -----------------------------
# Build stage
# -----------------------------
FROM rust:1.99.1-trixie AS builder
FROM rust:1.97.1-trixie AS builder
# Re-declare args after FROM
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETARCH
ARG RUSTFS_BUILD_FEATURES
ARG RC_RELEASE
# Debug: print platforms
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -54,7 +52,6 @@ RUN set -eux; \
build-essential \
ca-certificates \
curl \
jq \
git \
pkg-config \
libssl-dev \
@@ -169,8 +166,6 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
;; \
esac
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
# -----------------------------
# Development stage (keeps toolchain)
# -----------------------------
@@ -204,13 +199,10 @@ ENV RUSTFS_VOLUMES="/data" \
# We rely on cargo run to build and run
EXPOSE 9000 9001
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
chmod +x /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["cargo", "run", "--bin", "rustfs", "--"]
# -----------------------------
@@ -235,7 +227,6 @@ RUN set -eux; \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
jq \
tzdata \
coreutils; \
rm -rf /var/lib/apt/lists/*
@@ -257,11 +248,8 @@ RUN set -eux; \
# Copy the freshly built binary and the entrypoint
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
COPY bootstrap/ /etc/rustfs/bootstrap/
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
# Default environment (override in docker run/compose as needed)
ENV RUSTFS_VOLUMES="/data" \
@@ -275,5 +263,5 @@ EXPOSE 9000
VOLUME ["/data"]
# Keep root here; entrypoint will drop privileges using chroot --userspec
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
-21
View File
@@ -1,21 +0,0 @@
# RustFS image bootstrap
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
Optional per-bucket files use the bucket name as the filename without an extension:
| Directory | File format | Applied operation |
| --- | --- | --- |
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
-1
View File
@@ -1 +0,0 @@
[]
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-1
View File
@@ -1 +0,0 @@
-6
View File
@@ -1,6 +0,0 @@
{
"block_public_acls": true,
"ignore_public_acls": true,
"block_public_policy": true,
"restrict_public_buckets": true
}
-1
View File
@@ -1 +0,0 @@
+3 -3
View File
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
/// Default validation mode is "hop_by_hop".
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
/// Environment variable to enable RFC 7239 "Forwarded" header support.
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
/// RFC 7239 support is enabled by default.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
/// Environment variable for the maximum allowed proxy hops.
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
-6
View File
@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
/// Queue-store compression is enabled by default to reduce disk footprint.
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
+1 -6
View File
@@ -175,17 +175,12 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
/// Environment variable for the S3 request-body inter-chunk read timeout
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
/// (seconds). Default: 300. Set to 0 to disable.
///
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
/// while more bytes are still expected. It resets on every chunk, so it does not
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
/// partial body and then goes silent *without* closing the connection (no EOF).
/// Independently of this knob, `PutObject` and `UploadPart` require at least
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
/// that would otherwise pin upload admission indefinitely, and it stays in
/// force when this timeout is set to 0.
/// A reverse proxy or CDN that forwards a truncated body this way would
/// otherwise make RustFS wait forever for bytes that never arrive; with this
/// timeout the stalled read is aborted and logged with the received/expected
+3 -3
View File
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -502,163 +502,10 @@ mod tests {
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
"<Code>AccessDenied</Code>"
} else {
"<Code>BucketAlreadyOwnedByYou</Code>"
"<Code>BucketAlreadyExists</Code>"
}));
}
}
Ok(())
}
/// A deployment owns a single bucket namespace, so an IAM user that recreates
/// an existing bucket is the owner, not another account. In the default region
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
/// semantics), including buckets created by root and by another IAM user.
#[tokio::test]
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
-> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
let root = env.create_s3_client();
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readwrite",
"duplicate-iam",
)
.await?;
let iam = Client::from_conf(build_test_s3_config(
&env.url,
"duplicate-iam",
"duplicate-iam-secret",
None,
"duplicate-create",
));
root.create_bucket()
.bucket("duplicate-root-owned")
.send()
.await
.expect("root creates the first bucket");
iam.create_bucket()
.bucket("duplicate-root-owned")
.send()
.await
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
iam.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("the IAM user creates its own bucket");
iam.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("the same IAM user recreating its bucket gets 200 OK");
root.create_bucket()
.bucket("duplicate-iam-owned")
.send()
.await
.expect("root recreating an IAM-created bucket gets 200 OK");
Ok(())
}
/// Outside the default region the same ownership answer becomes
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
#[tokio::test]
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
let region = "eu-west-1";
let root = Client::from_conf(
env.create_s3_client()
.config()
.to_builder()
.region(Region::new(region))
.build(),
);
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readwrite",
"duplicate-eu-iam",
)
.await?;
let iam = Client::from_conf(
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
.to_builder()
.region(Region::new(region))
.build(),
);
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
admin_attach_user_policy_via(
AdminTransport::Signed,
&env.url,
&env.access_key,
&env.secret_key,
"readonly",
"duplicate-eu-readonly",
)
.await?;
let readonly = Client::from_conf(
build_test_s3_config(
&env.url,
"duplicate-eu-readonly",
"duplicate-eu-readonly-secret",
None,
"duplicate-create",
)
.to_builder()
.region(Region::new(region))
.build(),
);
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
client
.create_bucket()
.bucket(bucket)
.send()
.await
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
// Both the creator and the other authenticated identity own the namespace.
for creator in [&root, &iam] {
let error = creator
.create_bucket()
.bucket(bucket)
.send()
.await
.expect_err("a duplicate create outside us-east-1 must report ownership");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("BucketAlreadyOwnedByYou"),
"{bucket}"
);
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
}
}
let denied = readonly
.create_bucket()
.bucket("duplicate-eu-root")
.send()
.await
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
Ok(())
}
}
@@ -12,249 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::{
Client,
error::{ProvideErrorMetadata, SdkError},
primitives::ByteStream,
};
use std::any::TypeId;
use std::collections::BTreeSet;
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
use tokio::time::Instant;
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: Debug + ProvideErrorMetadata + 'static,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
// The workload uses unique unversioned keys. Repeat only explicitly
// advertised lock backpressure; every step still needs its real result.
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().is_some_and(|error| {
let Some(response) = error.raw_response() else {
return false;
};
let Some(service_error) = error.as_service_error() else {
return false;
};
// HEAD has no XML error body, so its SDK service error can
// lack a code even when the server supplies retry headers.
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|| (service_error.code().is_none()
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
response.status().as_u16() == 503
&& namespace_code
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
&& response.headers().get("retry-after") == Some("1")
});
if !retry {
return result;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await;
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
eprintln!(
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
started.elapsed()
);
}
outcome.map_err(|_| {
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
})
}
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
let actual = output.body.collect().await?.into_bytes();
if actual.as_ref() != expected {
return Err(format!(
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
expected.len(),
sha256_hex(expected),
actual.len(),
sha256_hex(&actual)
)
.into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_http_client::test_util::infallible_client_fn;
use aws_smithy_types::body::SdkBody;
use std::sync::atomic::{AtomicUsize, Ordering};
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
let requests = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&requests);
let transport = infallible_client_fn(move |_| {
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
let body = if code.is_empty() {
body.to_string()
} else {
format!("<Error><Code>{code}</Code></Error>")
};
let mut response = http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len());
if let Some(cause) = cause {
response = response.header("x-rustfs-error-cause", cause);
}
if let Some(retry_after) = retry_after {
response = response.header("retry-after", retry_after);
}
response.body(SdkBody::from(body)).expect("scripted S3 response")
});
let config = crate::common::build_test_s3_config(
"http://contention.example.invalid",
"synthetic-access",
"synthetic-secret",
None,
"contention-test",
);
let client = Client::from_conf(config.to_builder().http_client(transport).build());
(s3_probe_client(&client), requests)
}
fn contention() -> Reply {
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
}
#[tokio::test(start_paused = true)]
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt < 3 {
contention()
} else {
(200, "", None, None, "")
}
});
let standard = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
.build(),
);
standard
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
.await
.expect_err("three advertised lock responses exhaust the standard SDK policy");
assert_eq!(requests.load(Ordering::SeqCst), 3);
requests.store(0, Ordering::SeqCst);
let started = Instant::now();
after_lock_contention("PUT recovery", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await??;
assert_eq!(requests.load(Ordering::SeqCst), 4);
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt == 0 {
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
} else {
(200, "", None, None, "")
}
});
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
assert_eq!(requests.load(Ordering::SeqCst), 2);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
for reply in [
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(403, "AccessDenied", None, None, ""),
(404, "NoSuchKey", None, None, ""),
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "ServiceUnavailable", None, Some("1"), ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
] {
let (client, requests) = client(move |_| reply);
let result = after_lock_contention("PUT fatal response", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await?;
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn persistent_lock_contention_exhausts_the_request_budget() {
let (client, requests) = client(|_| contention());
let started = Instant::now();
let result = after_lock_contention("PUT unavailable", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await;
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
assert!(requests.load(Ordering::SeqCst) > 1);
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
}
#[tokio::test(start_paused = true)]
async fn acknowledged_wrong_bytes_fail_without_retry() {
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
.await
.expect_err("a successful but incorrect read is a consistency failure");
assert!(error.to_string().contains("bytes mismatch"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
// This consistency probe owns its admission policy; unrelated responses
// must not be hidden by the SDK's general-purpose retries.
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
let context = format!("worker={idx} round={round}");
after_lock_contention(&format!("{context} PUT source"), || {
writer
.put_object()
.bucket(&bucket)
.key(&key)
.body(ByteStream::from(body.clone()))
.send()
})
.await??;
put_object(writer, &bucket, &key, body.clone()).await?;
let head = after_lock_contention(&format!("{context} HEAD source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await??;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_workload_bytes(reader, &bucket, &key, &body).await?;
assert_object_bytes(reader, &bucket, &key, &body).await?;
after_lock_contention(&format!("{context} COPY retained"), || {
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
})
.await??;
assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
after_lock_contention(&format!("{context} DELETE source"), || {
writer.delete_object().bucket(&bucket).key(&key).send()
})
.await??;
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await?
.expect_err("deleted source key must not remain visible");
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
for handle in handles {
inventory.extend(handle.await??);
}
assert_eq!(
inventory.len(),
worker_count * rounds,
"every worker round must retain its acknowledged copy"
);
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
})
.await??;
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_workload_bytes(client, &bucket, key, body)
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
-3
View File
@@ -283,9 +283,6 @@ mod ssec_copy_test;
#[cfg(test)]
mod multipart_storage_class_test;
#[cfg(test)]
mod multipart_upload_id_test;
#[cfg(test)]
mod storage_class_capability_test;
@@ -1,175 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Multipart upload ID regression tests.
//!
//! Every operation that addresses an existing multipart upload must report an
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
//! the ID is malformed or is a well-formed ID of an aborted upload.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const BUCKET: &str = "multipart-upload-id";
const KEY: &str = "target.bin";
const COPY_SOURCE_KEY: &str = "copy-source.bin";
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
// the object layer rejects this ID as malformed instead of looking it up.
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
const PART_NUMBER: i32 = 1;
const PART_BODY: &[u8] = b"multipart upload id regression part";
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
where
E: ProvideErrorMetadata + std::fmt::Debug,
{
let Err(error) = result else {
return Some(format!("{operation}: succeeded"));
};
let status = error.raw_response().map(|response| response.status().as_u16());
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
}
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
env.create_test_bucket(BUCKET).await?;
let client = env.create_s3_client();
client
.put_object()
.bucket(BUCKET)
.key(COPY_SOURCE_KEY)
.body(ByteStream::from_static(PART_BODY))
.send()
.await?;
Ok((env, client))
}
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
let upload_part = client
.upload_part()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.body(ByteStream::from_static(PART_BODY))
.send()
.await;
let upload_part_copy = client
.upload_part_copy()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.part_number(PART_NUMBER)
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
.send()
.await;
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
let complete = client
.complete_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
.build(),
)
.send()
.await;
let abort = client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await;
let mismatches: Vec<String> = [
no_such_upload_mismatch("UploadPart", upload_part),
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
no_such_upload_mismatch("ListParts", list_parts),
no_such_upload_mismatch("CompleteMultipartUpload", complete),
no_such_upload_mismatch("AbortMultipartUpload", abort),
]
.into_iter()
.flatten()
.collect();
assert!(
mismatches.is_empty(),
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
mismatches.join("\n")
);
}
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
let status = head
.as_ref()
.err()
.and_then(|error| error.raw_response())
.map(|response| response.status().as_u16());
assert_eq!(
status,
Some(OBJECT_NOT_FOUND_STATUS),
"rejected operations must not publish {KEY}: {head:?}"
);
Ok(())
}
#[tokio::test]
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
assert_no_upload_or_object_published(&client).await
}
#[tokio::test]
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
let (_env, client) = start_server_with_copy_source().await?;
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
client
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(upload_id)
.send()
.await?;
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
assert_no_upload_or_object_published(&client).await
}
}
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
ServerSideEncryption, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
VersioningConfiguration,
};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
init_logging();
let mut source_env = RustFSTestEnvironment::new().await?;
let source_log = format!("{}/source.log", source_env.temp_dir);
source_env.capture_log_path = Some(source_log.clone());
let mut source_env_vars = replication_fast_env();
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
"target did not preserve the source delete-marker version ID"
);
let wait_for_source_writeback = || async {
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
timeout(Duration::from_secs(30), async {
loop {
let response = signed_request(
http::Method::GET,
&metrics_url,
&source_env.access_key,
&source_env.secret_key,
None,
None,
)
.await?;
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
let queued = metrics["queued"]["curr"]["count"]
.as_f64()
.ok_or("missing replication queue count")?;
if metrics["provider_available"] == true && queued == 0.0 {
return Ok::<(), Box<dyn Error + Send + Sync>>(());
}
sleep(Duration::from_millis(50)).await;
}
})
.await?
};
wait_for_source_writeback().await?;
let purged = source_client
.delete_objects()
source_client
.delete_object()
.bucket(source_bucket)
.delete(
Delete::builder()
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(&purged_version_id)
.build()?,
)
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(delete_marker_version_id)
.build()?,
)
.build()?,
)
.key(object_key)
.version_id(&purged_version_id)
.send()
.await?;
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
let target_state = list_replication_state(&target_client, target_bucket).await?;
assert!(
target_state.iter().all(|entry| entry.version_id != purged_version_id),
"target retained the explicitly purged object version"
);
assert!(
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
"target retained the explicitly purged delete marker"
);
assert!(
target_state
.iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
.await?;
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
// Wait for the source workers to finish their final writeback, which runs
// after the remote purge has already become visible (rustfs#9121).
wait_for_source_writeback().await?;
let logs = fs::read_to_string(source_log).await?;
let writeback_failures: Vec<_> = logs
.lines()
.filter(|line| line.contains("apply_replication_delete_state"))
.collect();
assert!(
writeback_failures.is_empty(),
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
);
Ok(())
}
-9
View File
@@ -94,12 +94,3 @@ pub(crate) mod scanner_peer_bucket {
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
pub(crate) use ecstore_rpc::PeerRestClient;
}
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
#[cfg(test)]
pub(crate) mod bucket_tag_fixture {
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
}
+29 -147
View File
@@ -360,26 +360,12 @@ impl OpaMock {
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, action, bucket, department);
Ok(())
}
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
let payload = self.next_request().await?;
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
assert_eq!(payload["input"]["resource"]["object"], object);
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
Ok(())
}
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
let input = &payload["input"];
assert_eq!(input["identity"]["account"], "opabuckettags");
assert_eq!(input["identity"]["is_owner"], false);
assert_eq!(input["action"], action);
assert_eq!(input["resource"]["bucket"], bucket);
let expected = department.map(|value| serde_json::json!([value]));
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
Ok(())
}
async fn wait_for_validation(&mut self) -> TestResult {
@@ -692,61 +678,22 @@ async fn test_list_buckets_opa_contract() -> TestResult {
Ok(())
}
/// Stop and reap the directory's owning server before opening the fixture store.
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
use crate::storage_api::bucket_tag_fixture::{
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
};
let server = env
.process
.as_mut()
.ok_or("missing server process before offline fixture write")?;
server.kill()?;
server.wait()?;
env.process = None;
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
let context = std::sync::Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
let _cancel = context
.background_cancel_token()
.expect("store binds its shutdown token")
.drop_guard();
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
let incarnation = metadata.bucket_incarnation_id;
let policy = metadata.policy_config_json.clone();
assert!(!incarnation.is_nil());
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
metadata.tagging_config_xml = xml.to_vec();
metadata.save_with_store(store.clone()).await?;
let persisted = load_bucket_metadata(store, bucket).await?;
assert_eq!(persisted.bucket_incarnation_id, incarnation);
assert_eq!(persisted.policy_config_json, policy);
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
Ok(())
}
#[tokio::test]
async fn test_bucket_tags_opa_contract() -> TestResult {
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
};
init_logging();
let mut opa = OpaMock::start().await?;
let mut env = RustFSTestEnvironment::new().await?;
let opa_url = opa.url.clone();
let server_env = [
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
env.start_rustfs_server_without_cleanup_with_env(&[
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
("NO_PROXY", "127.0.0.1,localhost"),
];
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
])
.await?;
let secret = uuid::Uuid::new_v4().to_string();
create_user(&env, "opabuckettags", &secret).await?;
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -786,7 +733,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let head = client
client
.head_object()
.bucket(source)
.key("report")
@@ -794,8 +741,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
assert_eq!(head.tag_count(), Some(1));
opa.expect_versioned_tag_permission(source, "report", version).await?;
client.list_objects_v2().bucket(source).send().await?;
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
client.list_object_versions().bucket(source).send().await?;
@@ -805,65 +750,28 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
admin.create_bucket().bucket(destination).send().await?;
set_department(&admin, destination, "engineering").await?;
let copy_source = format!("{source}/report?versionId={version}");
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("copied")
.send()
.await
.expect_err("denied COPY and REPLACE must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
}
let error = client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await
.expect_err("source tags must not authorize a different destination");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
.await?;
set_department(&admin, destination, "finance").await?;
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
// The access hook checks preliminary tags, then the copy use case checks
// the exact source reader's tags before publishing the destination.
for _ in 0..2 {
if directive == TaggingDirective::Copy {
opa.expect_versioned_tag_permission(source, "report", version).await?;
}
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
}
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
assert_eq!(tags.tag_set().len(), 1);
assert_eq!(tags.tag_set()[0].key(), "department");
assert_eq!(
tags.tag_set()[0].value(),
if directive == TaggingDirective::Copy {
"engineering"
} else {
"legal"
}
);
}
client
.copy_object()
.bucket(destination)
.key("copied")
.copy_source(&copy_source)
.send()
.await?;
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
let upload = client
.create_multipart_upload()
@@ -939,14 +847,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
.await?;
let missing = admin
.head_object()
.bucket(destination)
.key("denied-source")
.send()
.await
.expect_err("a denied source must leave no destination object");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
set_department(&client, source, "finance")
.await
.expect_err("requested tags must not authorize their own mutation");
@@ -985,7 +885,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
.await?;
let invalid_write = admin
admin
.put_bucket_tagging()
.bucket(destination)
.tagging(
@@ -995,25 +895,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
.build()?,
)
.send()
.await
.expect_err("new duplicate tags must be rejected before publication");
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
set_department(&admin, source, "finance").await?;
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
let read = client
.get_object()
.bucket(source)
.key("report")
.version_id(version)
.send()
.await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
let error = client
.get_object()
.bucket(destination)
-1
View File
@@ -31,7 +31,6 @@ workspace = true
[features]
default = []
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
# Sampled wall/CPU diagnostics only; does not change the read backend.
diagnostic-local-batch = ["rustix/time"]
+5 -16
View File
@@ -192,7 +192,6 @@ pub mod bucket {
table_catalog_path_hash, unreadable_config_refusal,
};
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
}
pub mod config_parse_mode {
@@ -332,8 +331,8 @@ pub mod bucket {
pub mod utils {
pub use crate::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
is_meta_bucketname, is_valid_object_prefix, serialize,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
serialize,
};
}
@@ -452,15 +451,9 @@ pub mod disk {
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
#[cfg(feature = "test-util")]
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
#[cfg(feature = "test-util")]
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
pub use crate::disk::multipart_index::{
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
MultipartIndexRpcRequest,
};
pub use crate::disk::multipart_source::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -473,9 +466,7 @@ pub mod disk {
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
@@ -626,7 +617,7 @@ pub mod rebalance {
pub mod rio {
pub use crate::io_support::rio::{
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
boxed_reader, compression_metadata_value, wrap_reader,
};
}
@@ -663,9 +654,7 @@ pub mod set_disk {
#[cfg(feature = "test-util")]
pub mod test_util {
pub use crate::bucket::quota::reservation::{
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
};
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
+5 -325
View File
@@ -16,8 +16,7 @@ use crate::bucket::metadata::BucketMetadata;
use crate::bucket::metadata_sys::get_bucket_targets_config;
use crate::bucket::metadata_sys::get_replication_config;
use crate::bucket::remote_s3_client::{
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
UploadResponseDeadline, build_remote_s3_client,
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
};
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -120,7 +119,6 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
connect_timeout: None,
read_timeout: None,
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
// Replication has no retry budget of its own on the request path,
// so it keeps the SDK's standard three attempts; stating it here
// pins the behaviour to this line instead of an SDK default.
@@ -1193,7 +1191,6 @@ impl BucketTargetSys {
health_check_duration: target.health_check_duration,
replicate_sync: target.replication_sync,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::from_env(),
})
}
@@ -1925,83 +1922,6 @@ impl std::fmt::Display for S3ClientError {
}
}
/// Replication request budgets, snapshotted when a target client is built.
/// Streamed GET bodies are not subject to the upload-response budget.
#[derive(Debug, Clone, Copy)]
pub struct TargetRequestTimeouts {
request: Duration,
complete: Duration,
minimum_transfer_rate: u64,
}
impl Default for TargetRequestTimeouts {
fn default() -> Self {
Self {
request: Duration::from_secs(60),
complete: Duration::from_secs(15 * 60),
minimum_transfer_rate: 1024 * 1024,
}
}
}
impl TargetRequestTimeouts {
fn from_env() -> Self {
let defaults = Self::default();
let seconds = |name: &str, fallback: Duration| {
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
Duration::from_secs(if (1..=86400).contains(&value) {
value
} else {
fallback.as_secs()
})
};
let minimum_transfer_rate =
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
Self {
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
minimum_transfer_rate: if minimum_transfer_rate == 0 {
defaults.minimum_transfer_rate
} else {
minimum_transfer_rate
},
}
}
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
// The SDK can make three attempts. Leave transmission headroom for
// all three; the separate response deadline still bounds a trickling
// target after each upload finishes.
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
RemoteS3RetryPolicy::Disabled => 1,
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
};
let seconds = size
.div_ceil(self.minimum_transfer_rate)
.checked_mul(attempts)
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
let timeout = self
.request
.checked_add(Duration::from_secs(seconds))
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
if Instant::now().checked_add(timeout).is_none() {
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
}
Ok(timeout)
}
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
Ok(aws_sdk_s3::config::Builder::new()
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.upload_timeout(size)?)
.build(),
)
.interceptor(UploadResponseDeadline(self.request)))
}
}
#[derive(Debug)]
pub struct TargetClient {
pub endpoint: String,
@@ -2015,7 +1935,6 @@ pub struct TargetClient {
pub health_check_duration: Duration,
pub replicate_sync: bool,
pub client: Arc<S3Client>,
pub timeouts: TargetRequestTimeouts,
}
impl TargetClient {
@@ -2441,11 +2360,12 @@ impl TargetClient {
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
});
let mut config = self.timeouts.upload_override(size)?;
if forwards_source_checksum {
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
operation = operation.config_override(
aws_sdk_s3::config::Builder::new()
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
);
}
operation = operation.config_override(config);
match operation.send().await {
Ok(output) => {
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2581,7 +2501,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(self.timeouts.upload_override(size)?)
.send()
.await
{
@@ -2619,13 +2538,6 @@ impl TargetClient {
}
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
})
.config_override(
aws_sdk_s3::config::Builder::new().timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(self.timeouts.complete)
.build(),
),
)
.send()
.await
{
@@ -3054,241 +2966,11 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_headers,
)
}
#[derive(Debug, Clone)]
struct TimedResponseConnector {
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: bytes::Bytes,
frames: Arc<std::sync::atomic::AtomicUsize>,
dropped: Arc<std::sync::atomic::AtomicUsize>,
}
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for ResponseDrop {
fn drop(&mut self) {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
impl SmithyHttpConnector for TimedResponseConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let settings = self.clone();
HttpConnectorFuture::new(async move {
http_body_util::BodyExt::collect(request.into_body())
.await
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
tokio::time::sleep(settings.upload_delay).await;
let guard = ResponseDrop(Arc::clone(&settings.dropped));
let stream = futures::stream::unfold(
(settings.prefix_bytes, Some(settings.response), true, guard),
move |(remaining, mut body, first, guard)| {
let frames = Arc::clone(&settings.frames);
async move {
if remaining == 0 && body.is_none() {
return None;
}
if !first {
tokio::time::sleep(settings.response_delay).await;
}
let bytes = if remaining > 0 {
bytes::Bytes::from_static(b" ")
} else {
body.take()?
};
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Some((
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
(remaining.saturating_sub(1), body, false, guard),
))
}
},
)
.fuse();
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
let mut response =
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
response.headers_mut().insert("etag", "\"test-etag\"");
Ok(response)
})
}
}
fn timed_response_target(
upload_delay: Duration,
response_delay: Duration,
prefix_bytes: usize,
response: &'static [u8],
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connector = SharedHttpConnector::new(TimedResponseConnector {
upload_delay,
response_delay,
prefix_bytes,
response: bytes::Bytes::from_static(response),
frames: Arc::clone(&frames),
dropped: Arc::clone(&dropped),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let (mut target, _) = header_recording_target_client(Vec::new());
target.timeouts = TargetRequestTimeouts {
request: Duration::from_secs(2),
complete: Duration::from_secs(10),
minimum_transfer_rate: 1024 * 1024,
};
target.client = Arc::new(S3Client::from_conf(
target
.client
.config()
.to_builder()
.http_client(http_client)
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
.timeout_config(
aws_smithy_types::timeout::TimeoutConfig::builder()
.operation_timeout(target.timeouts.request)
.build(),
)
.build(),
));
(target, frames, dropped)
}
#[test]
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
let policy = TargetRequestTimeouts::default();
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
assert_eq!(
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
Duration::from_secs(15420)
);
assert!(policy.upload_timeout(-1).is_err());
let slow = TargetRequestTimeouts {
minimum_transfer_rate: 1,
..policy
};
assert!(slow.upload_timeout(i64::MAX).is_err());
}
#[tokio::test(start_paused = true)]
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
for operation in ["put", "forwarded-checksum", "part"] {
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
let started = tokio::time::Instant::now();
let error = if operation == "part" {
target
.put_object_part(
"bucket",
"key",
"upload",
1,
size,
ByteStream::from(body),
&PutObjectPartOptions::default(),
)
.await
.expect_err("response deadline")
} else {
let mut opts = PutObjectOptions::default();
if operation == "forwarded-checksum" {
opts.user_metadata.insert(
"x-amz-checksum-sha256".to_owned(),
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
);
}
target
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
.await
.expect_err("response deadline")
};
assert!(
started.elapsed() >= Duration::from_secs(6),
"upload gets its full independent window: {error}"
);
assert!(
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
"response must end before the larger total deadline"
);
assert!(
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
"trickle delivered progress before failure"
);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
}
}
#[tokio::test(start_paused = true)]
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
let body = vec![b'x'; 2 * 1024 * 1024];
let size = i64::try_from(body.len()).expect("size");
target
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
.await
.expect("slow upload, prompt response");
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
let response = target
.get_object("bucket", "key", None, None, None, HeaderMap::new())
.await
.expect("GET headers");
let started = tokio::time::Instant::now();
let body = response
.body
.collect()
.await
.expect("streamed GET remains outside upload deadline")
.into_bytes();
assert!(started.elapsed() > target.timeouts.request);
assert_eq!(&body[..], b" payload");
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
for prefix in [5, 100] {
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
let started = tokio::time::Instant::now();
let result = target
.complete_multipart_upload(
"bucket",
"key",
"upload",
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
&PutObjectOptions::default(),
)
.await;
if prefix == 5 {
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
} else {
assert!(result.is_err());
assert!(started.elapsed() <= Duration::from_secs(11));
}
assert!(started.elapsed() > target.timeouts.request);
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
#[tokio::test(start_paused = true)]
async fn replication_control_operation_deadline_covers_response_trickle() {
let (target, frames, dropped) =
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
let started = tokio::time::Instant::now();
assert!(target.get_bucket_versioning("bucket").await.is_err());
assert!(started.elapsed() <= Duration::from_secs(3));
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
}
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3597,7 +3279,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
},
request_uris,
)
@@ -3910,7 +3591,6 @@ mod tests {
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
timeouts: TargetRequestTimeouts::default(),
})
}
@@ -5487,14 +5487,6 @@ pub async fn put_restore_opts(
if !oi.user_tags.is_empty() {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
}
if let Some(expires) = oi.expires {
meta.insert(
"expires".to_string(),
expires
.format(&time::format_description::well_known::Rfc3339)
.map_err(std::io::Error::other)?,
);
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
metadata_keys::RESTORE.to_string(),
@@ -5514,6 +5506,7 @@ pub async fn put_restore_opts(
// computed MD5 is not the object's public plaintext ETag.
preserve_etag: oi.etag.clone(),
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
//expires: oi.expires,
..Default::default()
})
}
@@ -7151,25 +7144,6 @@ mod tests {
0,
"tier generation lease should release after EOF"
);
let unread = get_transitioned_object_reader_with_tier_manager(
&object_info.bucket,
&object_info.name,
&None,
&HeaderMap::new(),
&object_info,
&ObjectOptions::default(),
&manager,
None,
)
.await
.expect("open an unread transitioned response");
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
drop(unread);
assert_eq!(
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
0,
"canceling an unread response must release the tier lease"
);
}
#[cfg(feature = "test-util")]
-71
View File
@@ -256,38 +256,6 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
/// Shared tag-entry validation for object requests and persisted bucket tags.
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
fn is_valid_tag_text(value: &str) -> bool {
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
VALID_TAG_TEXT
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
.as_ref()
.is_ok_and(|pattern| pattern.is_match(value))
}
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
for tag in tag_set {
let key = tag
.key
.as_deref()
.filter(|key| !key.is_empty())
.ok_or("Tag key cannot be empty")?;
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
return Err("The TagKey you have provided is invalid");
}
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
return Err("The TagValue you have provided is invalid");
}
if !keys.insert(key) {
return Err("Cannot provide multiple Tags with the same key");
}
}
Ok(())
}
pub const MAX_BUCKET_TAGS: usize = 50;
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -1022,20 +990,6 @@ impl BucketMetadata {
self.encryption_config_updated_at = updated;
}
BUCKET_TAGGING_CONFIG => {
if !data.is_empty() {
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
})?;
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
return Err(Error::InvalidArgument(
self.name.clone(),
String::new(),
"Cannot have more than 50 tags per bucket".to_owned(),
));
}
validate_tag_set_entries(&tagging.tag_set)
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
}
self.tagging_config_xml = data;
// Drop the parsed form (like lifecycle above) so clearing the
// payload can't leave stale parsed tags to be cached.
@@ -1830,31 +1784,6 @@ mod test {
/// backlog#2292: a replicated config is stamped with the source
/// `updated_at` it was given, not the local clock, while the plain
/// `update_config` entry keeps stamping the local clock.
#[test]
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
let mut metadata = BucketMetadata::new("validated-tags");
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
.expect("empty value is valid");
let updated = metadata.tagging_config_updated_at;
for invalid in [
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
] {
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
assert_eq!(metadata.tagging_config_xml, valid);
assert_eq!(metadata.tagging_config_updated_at, updated);
}
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
metadata
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
.expect("deletion repairs legacy corruption");
assert!(metadata.tagging_config_xml.is_empty());
}
#[test]
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
+5 -11
View File
@@ -3428,7 +3428,7 @@ mod tests {
),
(
BUCKET_TAGGING_CONFIG,
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
),
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
(
@@ -4556,13 +4556,8 @@ mod tests {
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
}
let mut bm = BucketMetadata::new(bucket);
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
// Model legacy persisted corruption without passing the write-time validator.
bm.tagging_config_xml = raw.to_vec();
} else {
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
}
bm.update_config(config_file, raw.to_vec())
.expect("raw config should be stored");
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
}
@@ -5743,8 +5738,7 @@ mod tests {
// update only needs one interleaving to show up.
const ROUNDS: usize = 8;
for round in 0..ROUNDS {
let tagging =
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5817,7 +5811,7 @@ mod tests {
let blocked = tokio::spawn({
let node_b = node_b.clone();
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
});
// Long enough for the write to have finished had it not waited: the
@@ -119,21 +119,6 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
// Exercise the real storage entry point before testing the new
// reporting semantics: the same ledger must admit and refuse
// these writes even when a scanner floor claims a million bytes.
if protocol == 1 {
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
assert!(
matches!(
store
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
.await,
Err(StorageError::SlowDown)
),
"a missing authoritative ledger must not admit a write from scanner reporting"
);
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
.await
.expect("bootstrap authoritative usage through the production worker");
}
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
assert!(
matches!(
+49 -739
View File
@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{DiskAPI, error::DiskError};
use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::namespace::NamespaceLocking;
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
use rustfs_lock::NamespaceLockGuard;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashSet};
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use time::OffsetDateTime;
@@ -36,8 +36,6 @@ use tracing::warn;
use uuid::Uuid;
const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
const RECONCILE_RETRY_SECONDS: i64 = 30;
const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
@@ -98,10 +96,6 @@ struct QuotaLedger {
version: u8,
bucket_incarnation: Uuid,
quota_revision_unix_nanos: i128,
/// Changes on every durable write, including zero-growth reservations.
/// Missing on an older writer's output, which invalidates a concurrent scan.
#[serde(default)]
writer_revision: Uuid,
accounted_usage: u64,
reservations: BTreeMap<Uuid, PersistedReservation>,
#[serde(default)]
@@ -116,7 +110,6 @@ impl QuotaLedger {
version: QUOTA_LEDGER_FORMAT_VERSION,
bucket_incarnation,
quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
writer_revision: Uuid::nil(),
accounted_usage,
reservations: BTreeMap::new(),
reconcile_required: false,
@@ -492,23 +485,22 @@ impl QuotaContext {
let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
let mut ledger =
load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
.await?;
let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
let Some(mut ledger) = ledger.take() else {
// Bootstrap has no authoritative usage yet. Release every
// request-owned fence by returning; the worker owns no request.
job.schedule();
return Err(StorageError::SlowDown);
};
let mut ledger = load_current_ledger_locked(
Arc::clone(&store),
&ledger_data.bucket,
&ledger_data.ledger_object,
bucket_incarnation,
quota_revision,
)
.await?;
let growth = reservation.growth();
let current_usage = ledger.admitted_usage()?;
let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
if ledger.reconcile_required
|| (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
{
job.schedule();
let mut current_usage = ledger.admitted_usage()?;
let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
current_usage = ledger.admitted_usage()?;
expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
}
if growth > 0 && expected_usage > quota_limit {
return Err(StorageError::QuotaExceeded {
@@ -526,7 +518,7 @@ impl QuotaContext {
});
}
ledger.reserve(operation_id, reservation)?;
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
Ok(QuotaReservation {
ledger: Some(ReservationLedgerData::Single(ledger_data)),
@@ -1090,7 +1082,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
ledger.mark_commit_started(data.operation_id, &data.reservation)?;
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
})
.await
.map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
@@ -1121,7 +1113,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
} else {
ledger.abort(operation_id, &reservation)?;
}
save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
})
.await
.map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
@@ -1129,15 +1121,22 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
async fn load_current_ledger_locked(
store: Arc<ECStore>,
bucket: &str,
ledger_object: &str,
bucket_incarnation: Uuid,
quota_revision: OffsetDateTime,
) -> Result<Option<QuotaLedger>> {
match load_ledger_locked(store, ledger_object).await {
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
) -> Result<QuotaLedger> {
match load_ledger_locked(Arc::clone(&store), ledger_object).await {
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
let usage = exact_bucket_usage(&store, bucket).await?;
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
}
Ok(_) => Err(StorageError::PartMissingOrCorrupt),
Err(StorageError::ConfigNotFound) => Ok(None),
Err(StorageError::ConfigNotFound) => {
let usage = exact_bucket_usage(&store, bucket).await?;
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
}
Err(err) => Err(err),
}
}
@@ -1246,11 +1245,15 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
for (object, target) in orphan_commit_targets {
fence_namespace_mutations(&store, bucket, &object, target).await?;
}
let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
if ledger.reservations.is_empty() && ledger.reconcile_required {
reconcile_exact(&store, bucket, &mut ledger).await?;
removed = true;
}
if !removed && !cursor_changed {
return Ok(());
}
save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
}
fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
@@ -1270,217 +1273,13 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
Ok(removed)
}
#[derive(Debug, Serialize, Deserialize)]
struct ReconcileControl {
bucket_incarnation: Uuid,
quota_revision_unix_nanos: i128,
next_attempt_at: i64,
}
#[derive(Clone)]
struct ReconcileJob {
store: Arc<ECStore>,
bucket: String,
bucket_incarnation: Uuid,
quota_revision: OffsetDateTime,
}
type ReconcileKey = (Uuid, String, Uuid, i128);
fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
ACTIVE.get_or_init(Default::default)
}
struct ActiveReconciliation(ReconcileKey);
impl Drop for ActiveReconciliation {
fn drop(&mut self) {
active_reconciliations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&self.0);
}
}
impl ReconcileJob {
fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
Self {
store,
bucket,
bucket_incarnation,
quota_revision,
}
}
fn schedule(self) {
let key = (
self.store.id,
self.bucket.clone(),
self.bucket_incarnation,
self.quota_revision.unix_timestamp_nanos(),
);
if !active_reconciliations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(key.clone())
{
return;
}
let owner = ActiveReconciliation(key);
// The owner carries no object, metadata, lifecycle or reservation guard.
// Cancellation of the rejected request cannot cancel its reconciliation.
tokio::spawn(async move {
let _owner = owner;
if let Err(error) = self.run().await {
warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
error = %error, "Quota reconciliation deferred");
}
});
}
fn control_object(&self) -> String {
format!(
"{}.reconcile/{}/{}",
ledger_object(&self.bucket),
self.bucket_incarnation,
self.quota_revision.unix_timestamp_nanos()
)
}
async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
if guard.is_lock_lost() {
return Err(StorageError::SlowDown);
}
let mut opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let _ = opts.set_quota_admission(0, u64::MAX);
opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
}
async fn run(&self) -> Result<()> {
let control_object = self.control_object();
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
return Ok(());
};
let guard = Arc::new(guard);
let now = now_unix();
match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
Ok(bytes) => {
let control: ReconcileControl = serde_json::from_slice(&bytes)?;
if control.bucket_incarnation != self.bucket_incarnation
|| control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
{
return Err(StorageError::PartMissingOrCorrupt);
}
if control.next_attempt_at > now {
return Ok(());
}
}
Err(StorageError::ConfigNotFound) => {}
Err(error) => return Err(error),
}
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
let mut control = ReconcileControl {
bucket_incarnation: self.bucket_incarnation,
quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
};
// Persist the throttle before work, including bootstrap and failed scans.
self.save_control(&control_object, &control, &guard).await?;
let result = self.scan_and_publish(&guard).await;
control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
self.save_control(&control_object, &control, &guard).await?;
result
}
/// Take a short authoritative configuration snapshot without recursively
/// acquiring a reader behind a queued metadata/lifecycle writer.
async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
let guard = snapshot
.metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
.ok_or(StorageError::PreconditionFailed)?;
let opts = ObjectOptions {
expected_bucket_incarnation_id: Some(self.bucket_incarnation),
object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
..Default::default()
};
let (quota, incarnation, revision) =
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
.await?;
if incarnation != self.bucket_incarnation
|| revision != self.quota_revision
|| !quota.as_ref().is_some_and(|quota| {
quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
})
{
return Err(StorageError::PreconditionFailed);
}
Ok(snapshot)
}
async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
let ledger_object = ledger_object(&self.bucket);
let expected_revision = {
// Lock order: reconcile owner, lifecycle, metadata, ledger.
// Request paths never wait for the reconcile owner.
let _configuration = self.configuration_snapshot().await?;
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
Ok(mut ledger) => {
if !ledger.reservations.is_empty() {
return Ok(());
}
// Seed a non-nil token even for a legacy ledger. An old
// node drops this field on write and cannot pass the CAS.
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
Some(ledger.writer_revision)
}
Err(StorageError::ConfigNotFound) => None,
Err(error) => return Err(error),
}
};
#[cfg(test)]
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
// Only the independent reconcile owner is held during this whole-bucket
// walk. Concurrent reserve/settle/reap writes invalidate its result.
let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
#[cfg(test)]
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
let configuration = self.configuration_snapshot().await?;
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
Ok(ledger) => {
expected_revision == Some(ledger.writer_revision)
&& !ledger.writer_revision.is_nil()
&& ledger.reservations.is_empty()
}
Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
Err(error) => return Err(error),
};
if !unchanged {
return Ok(());
}
if owner.is_lock_lost()
|| !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
{
return Err(StorageError::SlowDown);
}
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
if !ledger.reservations.is_empty() {
return Err(StorageError::PartMissingOrCorrupt);
}
ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
ledger.reconcile_required = false;
Ok(())
}
async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
@@ -2136,7 +1935,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
async fn save_ledger_locked(
store: Arc<ECStore>,
ledger_object: &str,
ledger: &mut QuotaLedger,
ledger: &QuotaLedger,
ledger_guard: &Arc<NamespaceLockGuard>,
) -> Result<()> {
if ledger_guard.is_lock_lost() {
@@ -2160,71 +1959,9 @@ async fn save_ledger_locked(
let _ = opts.set_quota_admission(0, u64::MAX);
opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
ledger.writer_revision = Uuid::new_v4();
save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
}
/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
/// The production reaper, ownership fence, scan and compare-and-swap still run.
#[cfg(any(test, feature = "test-util"))]
pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let active = active_reconciliations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains(&key);
if !active {
break;
}
tokio::task::yield_now().await;
}
})
.await
.map_err(|_| StorageError::SlowDown)?;
reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
let object = job.control_object();
{
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
match read_config_no_lock(Arc::clone(&store), &object).await {
Ok(bytes) => {
let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
control.next_attempt_at = 0;
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
job.save_control(&object, &control, &guard).await?;
}
Err(StorageError::ConfigNotFound) => {}
Err(error) => return Err(error),
}
}
job.run().await
}
/// Bootstrap a test bucket through the real background reconciliation path.
/// Fixtures call this after enabling V1 quota, before issuing writes whose
/// contract is independent of the initial retryable admission response.
#[cfg(any(test, feature = "test-util"))]
pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
if !quota
.as_ref()
.is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
{
return Ok(());
}
ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
.run()
.await?;
match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
Some(_) => Ok(()),
None => Err(StorageError::SlowDown),
}
}
#[cfg(any(test, feature = "test-util"))]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn fail_next_quota_ledger_save_for_test() {
@@ -2266,434 +2003,6 @@ mod tests {
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use serial_test::serial;
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum ReconcilePauseStage {
BeforeScan,
BeforePublish,
}
struct ReconcilePause {
bucket: String,
stage: ReconcilePauseStage,
armed: std::sync::atomic::AtomicBool,
scans: std::sync::atomic::AtomicUsize,
fail_scan: std::sync::atomic::AtomicBool,
reached: tokio::sync::Notify,
release: tokio::sync::Notify,
}
static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
struct ReconcilePauseGuard(Arc<ReconcilePause>);
impl ReconcilePauseGuard {
fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
let pause = Arc::new(ReconcilePause {
bucket: bucket.to_owned(),
stage,
armed: true.into(),
scans: 0.into(),
fail_scan: false.into(),
reached: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
});
*RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
Self(pause)
}
async fn reached(&self) {
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
.await
.expect("reconciliation reached barrier");
}
}
impl Drop for ReconcilePauseGuard {
fn drop(&mut self) {
self.0.release.notify_one();
let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
*hook = None;
}
}
}
pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
use std::sync::atomic::Ordering;
let hook = RECONCILE_PAUSE
.lock()
.expect("reconcile hooks")
.as_ref()
.filter(|hook| hook.bucket == bucket)
.cloned();
if let Some(hook) = hook {
if stage == ReconcilePauseStage::BeforeScan {
hook.scans.fetch_add(1, Ordering::SeqCst);
}
if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
hook.reached.notify_one();
hook.release.notified().await;
}
if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
return Err(StorageError::other("injected quota usage failure"));
}
}
Ok(())
}
async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
use crate::object_api::PutObjReader;
use crate::storage_api_contracts::object::ObjectIO;
let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let mut seed = PutObjReader::from_vec(vec![1; 10]);
store
.put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
.await
.expect("existing usage");
store
.update_bucket_metadata_config(
&bucket,
"quota.json",
serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
)
.await
.expect("enable quota");
let (_, incarnation, revision) = store
.bucket_quota_and_incarnation_from_disk(&bucket)
.await
.expect("quota identity");
(directories, ReconcileJob::new(store, bucket, incarnation, revision))
}
async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
.await?
.reserve(old, new)
.await
}
async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
.await
.expect("ledger")
}
async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
let key = (
job.store.id,
job.bucket.clone(),
job.bucket_incarnation,
job.quota_revision.unix_timestamp_nanos(),
);
tokio::time::timeout(Duration::from_secs(10), async {
while active_reconciliations().lock().expect("active workers").contains(&key) {
tokio::task::yield_now().await;
}
})
.await
.expect("owned reconciliation finished");
}
async fn expire_reconciliation_backoff(job: &ReconcileJob) {
drain_reconciliation_scheduler(job).await;
let object = job.control_object();
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
let mut control: ReconcileControl =
serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
.expect("control JSON");
control.next_attempt_at = 0;
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
.await
.expect("control fence");
job.save_control(&object, &control, &guard).await.expect("advance test clock");
}
async fn wait_for_reconcile_owner(job: &ReconcileJob) {
let object = job.control_object();
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
.await
.expect("worker finished")
.expect("owner guard");
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
use std::sync::atomic::Ordering;
let (_directories, job) = reconciliation_fixture(10).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
assert!(
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
"unknown usage cannot admit a reservation"
);
pause.reached().await;
assert!(
matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
),
"bootstrap must not publish zero usage"
);
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
// Bypass the local scheduler, exercising the cross-node ownership boundary.
tokio::time::timeout(Duration::from_secs(2), job.run())
.await
.expect("contender must not wait for a scan")
.expect("another owner is active");
let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
.await
.expect("scan does not retain lifecycle readers")
.expect("lifecycle writer");
let metadata = tokio::time::timeout(
Duration::from_secs(2),
metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
)
.await
.expect("scan does not retain metadata readers")
.expect("metadata writer");
drop(metadata);
drop(lifecycle);
pause.0.release.notify_one();
wait_for_reconcile_owner(&job).await;
let ledger = read_test_ledger(&job).await;
assert_eq!(ledger.accounted_usage, 10);
assert!(ledger.reservations.is_empty());
assert!(!ledger.writer_revision.is_nil());
for _ in 0..3 {
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
job.run().await.expect("persisted cooldown");
}
drain_reconciliation_scheduler(&job).await;
assert_eq!(
pause.0.scans.load(Ordering::SeqCst),
1,
"denied requests cannot repeatedly scan a full bucket"
);
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
let (_directories, job) = reconciliation_fixture(100).await;
job.run().await.expect("bootstrap");
for mutation in ["abort", "commit", "reap", "legacy"] {
expire_reconciliation_backoff(&job).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
let worker = tokio::spawn({
let job = job.clone();
async move { job.run().await }
});
pause.reached().await;
let initial = read_test_ledger(&job).await.writer_revision;
if mutation == "legacy" {
let object = ledger_object(&job.bucket);
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
document.as_object_mut().expect("ledger map").remove("writer_revision");
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
.await
.expect("ledger fence");
let mut opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let _ = opts.set_quota_admission(0, u64::MAX);
opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
save_config_with_opts(
Arc::clone(&job.store),
&object,
serde_json::to_vec(&document).expect("old node JSON"),
&opts,
)
.await
.expect("old node drops new field");
} else {
let mut reservation = quota_reservation(&job, 10, 10)
.await
.expect("zero-growth admission while scanning");
let reserved = read_test_ledger(&job).await.writer_revision;
assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
reservation.mark_commit_started().await.expect("commit marker");
assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
match mutation {
"abort" => reservation.abort().await,
"commit" => reservation.commit().await,
"reap" => {
reservation.defer_after_fence();
reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
.await
.expect("reap orphan");
}
_ => unreachable!(),
}
}
let written = read_test_ledger(&job).await;
assert!(written.reservations.is_empty());
assert_ne!(written.writer_revision, initial);
pause.0.release.notify_one();
worker.await.expect("worker join").expect("stale scan discarded");
let current = read_test_ledger(&job).await;
assert_eq!(
current.writer_revision, written.writer_revision,
"{mutation}: stale usage must not replace a concurrent writer"
);
assert_eq!(current.accounted_usage, written.accounted_usage);
}
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
use std::sync::atomic::Ordering;
let (_directories, job) = reconciliation_fixture(10).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
pause.0.fail_scan.store(true, Ordering::SeqCst);
pause.0.release.notify_one();
assert!(job.run().await.is_err());
assert!(matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
));
job.run().await.expect("failure cooldown");
assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
pause.0.fail_scan.store(false, Ordering::SeqCst);
expire_reconciliation_backoff(&job).await;
job.run().await.expect("recover after failure");
assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
job.store
.delete_object(&job.bucket, "seed", ObjectOptions::default())
.await
.expect("delete stored usage");
assert!(
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
"stale usage remains conservative during cooldown"
);
expire_reconciliation_backoff(&job).await;
job.run().await.expect("refresh deleted usage");
assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
quota_reservation(&job, 0, 1)
.await
.expect("delete must not leave bucket permanently full")
.abort()
.await;
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
let (_directories, job) = reconciliation_fixture(100).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
let caller = tokio::spawn({
let job = job.clone();
async move {
job.schedule();
std::future::pending::<()>().await
}
});
pause.reached().await;
caller.abort();
assert!(caller.await.expect_err("cancel caller").is_cancelled());
job.store
.update_bucket_metadata_config(
&job.bucket,
"quota.json",
serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
)
.await
.expect("quota changes while scanning");
pause.0.release.notify_one();
wait_for_reconcile_owner(&job).await;
assert!(
matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
),
"old config cannot publish bootstrap usage"
);
drop(pause);
let (_, incarnation, revision) = job
.store
.bucket_quota_and_incarnation_from_disk(&job.bucket)
.await
.expect("current configuration");
let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
let caller = tokio::spawn({
let job = current.clone();
async move {
job.schedule();
std::future::pending::<()>().await
}
});
pause.reached().await;
caller.abort();
assert!(caller.await.expect_err("cancel new caller").is_cancelled());
pause.0.release.notify_one();
wait_for_reconcile_owner(&current).await;
assert!(
read_test_ledger(&current).await.matches(incarnation, revision),
"owned scan publishes after its waiter is canceled"
);
}
#[tokio::test]
#[serial]
async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
use crate::storage_api_contracts::bucket::DeleteBucketOptions;
let (_directories, job) = reconciliation_fixture(100).await;
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
let worker = tokio::spawn({
let job = job.clone();
async move { job.run().await }
});
pause.reached().await;
job.store
.delete_object(&job.bucket, "seed", ObjectOptions::default())
.await
.expect("empty old bucket");
job.store
.delete_bucket(&job.bucket, &DeleteBucketOptions::default())
.await
.expect("retire bucket while scan is paused");
job.store
.make_bucket(&job.bucket, &MakeBucketOptions::default())
.await
.expect("replacement bucket");
job.store
.update_bucket_metadata_config(
&job.bucket,
"quota.json",
serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
)
.await
.expect("replacement quota");
pause.0.release.notify_one();
assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
assert!(matches!(
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
Err(StorageError::ConfigNotFound)
));
drop(pause);
initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
.await
.expect("replacement can bootstrap independently");
let replacement = read_test_ledger(&job).await;
assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
assert_eq!(replacement.accounted_usage, 0);
}
fn ledger(accounted_usage: u64) -> QuotaLedger {
QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
}
@@ -2781,7 +2090,8 @@ mod tests {
.expect("persist crash-left reservation fixture");
assert!(matches!(
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
.await,
Err(StorageError::PartMissingOrCorrupt)
));
assert!(matches!(
@@ -2799,10 +2109,10 @@ mod tests {
Err(StorageError::ConfigNotFound)
));
let replacement =
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
.await
.expect("new quota admission must not inherit a fresh stale reservation");
assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
assert!(replacement.reservations.is_empty());
}
#[test]
+2 -87
View File
@@ -40,9 +40,7 @@ use aws_smithy_runtime_api::client::http::{
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
};
use aws_smithy_runtime_api::client::interceptors::Intercept;
use aws_smithy_runtime_api::client::interceptors::context::{
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
};
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
use aws_smithy_runtime_api::client::result::ConnectorError;
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -157,8 +155,6 @@ pub struct RemoteS3EndpointSpec {
pub ca_cert_pem: Option<String>,
pub connect_timeout: Option<Duration>,
pub read_timeout: Option<Duration>,
/// Entire SDK call, including non-streaming response bodies and retries.
pub operation_timeout: Option<Duration>,
/// How many wire requests one logical call may cost. Every consumer
/// declares it; see [`RemoteS3RetryPolicy`].
pub retry: RemoteS3RetryPolicy,
@@ -256,78 +252,6 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
Ok(builder.build())
}
/// Only installed on non-streaming upload operations. The budget begins at
/// response headers, independently of the potentially long request upload.
#[derive(Debug)]
pub(crate) struct UploadResponseDeadline(pub Duration);
impl Intercept for UploadResponseDeadline {
fn name(&self) -> &'static str {
"RustfsUploadResponseDeadline"
}
fn modify_before_deserialization(
&self,
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
_runtime_components: &RuntimeComponents,
_cfg: &mut ConfigBag,
) -> Result<(), BoxError> {
let response = context.response_mut();
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
inner: Some(inner),
deadline: Box::pin(tokio::time::sleep(self.0)),
});
Ok(())
}
}
struct UploadResponseBody {
inner: Option<SdkBody>,
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
}
impl http_body::Body for UploadResponseBody {
type Data = bytes::Bytes;
type Error = BoxError;
fn poll_frame(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
use std::future::Future as _;
use std::task::Poll;
if self.inner.is_none() {
return Poll::Ready(None);
}
// The absolute response budget never renews on data, empty frames,
// or trailers. A retry gets its own bounded response window.
if self.deadline.as_mut().poll(cx).is_ready() {
self.inner.take();
return Poll::Ready(Some(Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"replication response body deadline exceeded",
)
.into())));
}
let Some(inner) = self.inner.as_mut() else {
return Poll::Ready(None);
};
let result = std::pin::Pin::new(inner).poll_frame(cx);
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
self.inner.take();
}
result
}
fn is_end_stream(&self) -> bool {
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
}
fn size_hint(&self) -> http_body::SizeHint {
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
}
}
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
/// signature stays valid.
@@ -385,7 +309,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
config_builder = config_builder.http_client(http_client);
}
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
let mut timeouts = TimeoutConfig::builder();
if let Some(connect_timeout) = spec.connect_timeout {
timeouts = timeouts.connect_timeout(connect_timeout);
@@ -393,9 +317,6 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
if let Some(read_timeout) = spec.read_timeout {
timeouts = timeouts.read_timeout(read_timeout);
}
if let Some(operation_timeout) = spec.operation_timeout {
timeouts = timeouts.operation_timeout(operation_timeout);
}
config_builder = config_builder.timeout_config(timeouts.build());
}
@@ -753,7 +674,6 @@ mod tests {
ca_cert_pem: None,
connect_timeout: None,
read_timeout: None,
operation_timeout: None,
retry: RemoteS3RetryPolicy::Disabled,
user_agent_suffix: "",
}
@@ -1107,16 +1027,11 @@ mod tests {
spec.user_agent_suffix = "RustFS-Test/0.0";
spec.connect_timeout = Some(Duration::from_secs(5));
spec.read_timeout = Some(Duration::from_secs(5));
spec.operation_timeout = Some(Duration::from_secs(12));
let config = build_remote_s3_config(&spec)
.await
.expect("spec should build")
.http_client(http_client)
.build();
assert_eq!(
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
Some(Duration::from_secs(12))
);
S3Client::from_conf(config)
.head_bucket()
.bucket("bucket")
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
impl ReplicationMetadataStore {
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
@@ -117,10 +116,6 @@ mod tests {
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
"config/replication/mrf.bin.recovery"
);
assert_eq!(
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
"config/replication/mrf.bin.replay"
);
assert_eq!(
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
"config/replication/force-delete.bin"
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
use super::replication_filemeta_boundary::{
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
replication_statuses_map, version_purge_statuses_map,
};
use super::replication_lock_boundary::ReplicationLockTiming;
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
const MRF_PENDING_CAP: usize = 200_000;
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
/// re-attempts every retained entry, so a target that stays down costs one
/// attempt per entry per interval.
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
#[derive(Debug, Default)]
pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
}
/// Acknowledge only the generation read by the recovery leader. Entries appended
/// after that generation are retained as a suffix and replayed on the next pass.
/// after that generation are retained as a suffix and replayed on the next startup.
/// Lock order: recovery leader lock -> MRF journal object lock.
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
Err(EcstoreError::PreconditionFailed)
}
/// Acquires the MRF recovery leader lock for one journal replay pass.
/// Acquires the MRF recovery leader lock for the startup replay.
/// Returns `None` (after logging) when the lock cannot be created or another
/// node is already processing the backlog.
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
}
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
/// On acknowledgement failure the backlog is preserved for the next replay pass and
/// On acknowledgement failure the backlog is preserved for the next startup and
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
}
}
fn unix_millis_now() -> i64 {
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
}
/// Whether a periodic pass may run, given the start time of the last pass on
/// any node. A stamp more than one interval ahead of this node's clock is skew
/// or a bad value, so it does not suppress replay.
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
let Some(last_started_ms) = last_started_ms else {
return true;
};
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
}
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
.await
.ok()?;
std::str::from_utf8(&data).ok()?.trim().parse().ok()
}
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
if let Err(error) = ReplicationConfigStore::save(
storage.clone(),
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
started_ms.to_string().into_bytes(),
)
.await
{
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to record the MRF replay start time"
);
}
}
/// One MRF journal replay pass. The recovery leader reads the journal,
/// re-drives every entry, and acknowledges the replayed prefix; entries that
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
///
/// Every node runs the periodic loop, and the leader lock only serializes
/// passes. With `min_interval`, a pass is skipped when a pass on any node
/// started within that interval, read from the shared replay stamp, so a
/// cluster replays once per interval rather than once per node. The startup
/// pass passes `None` and always runs. A pass that finds entries records its
/// start time. Returns whether the journal was read.
/// Lock order: recovery leader lock -> replay stamp object lock.
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
return false;
};
let started_ms = unix_millis_now();
if let Some(interval) = min_interval
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
{
return false;
}
let Some(entries) = load_mrf_recovery_entries(storage).await else {
return true;
};
if !entries.is_empty() {
write_mrf_replay_stamp(storage, started_ms).await;
}
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
true
}
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
/// one pass at a time across the cluster, including the startup pass, and the
/// shared replay stamp spaces passes from different nodes by `period`.
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
tokio::spawn(async move {
loop {
tokio::time::sleep(period).await;
replay_mrf_journal(&storage, Some(period)).await;
}
})
}
#[derive(Debug, thiserror::Error)]
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Start background tasks
pool.start_mrf_persister().await;
pool.start_mrf_processor().await;
pool.start_mrf_replayer().await;
pool.start_force_delete_processor().await;
pool
@@ -1581,7 +1462,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
// A source generation already queued or being uploaded is not driven a
// A version that is already queued or being uploaded is not driven a
// second time: the scanner heal pass sees it as PENDING until the
// first upload lands and would otherwise re-queue it every cycle
// (backlog#2362). The key is released when the worker finishes, or
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Starts the MRF processor — one-shot at startup.
///
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
/// the pass while the server runs.
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
/// rewrites any entries that could not be admitted for a later startup retry.
async fn start_mrf_processor(&self) {
let storage = self.storage.clone();
let handle = tokio::spawn(async move {
replay_mrf_journal(&storage, None).await;
});
self.task_handles.lock().await.push(handle);
}
/// Starts the runtime MRF replayer — ongoing background task.
///
/// Entries reach the journal while the server runs (worker-queue overflow,
/// delete-marker purge intents whose watch window drained with targets still
/// dirty). Replaying only at startup left them waiting for a restart.
async fn start_mrf_replayer(&self) {
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
let handle = tokio::spawn(async move {
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
return;
};
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
return;
};
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
});
self.task_handles.lock().await.push(handle);
}
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// writes them to the on-disk MRF file every flush interval (default 10s,
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
/// entries accumulate. Runtime writers append under the journal lock; only the
/// recovery leader of a replay pass may remove an acknowledged prefix.
/// startup recovery leader may remove an acknowledged prefix.
async fn start_mrf_persister(&self) {
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
return;
@@ -2404,47 +2314,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
/// Object versions currently queued or being uploaded, keyed by bucket,
/// object name, version and source generation. `queue_replica_task` admits a generation only once
/// object name and version. `queue_replica_task` admits a version only once
/// while it is in flight; the scanner heal pass and MRF replays that arrive
/// in the meantime are `Skipped` instead of driving a second complete upload
/// (backlog#2362). Entries are removed when the worker finishes the task or
/// when no worker accepted it.
#[derive(Debug, Default)]
pub(crate) struct ReplicationInFlight {
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ReplicationInFlightKey {
bucket: String,
object: String,
version_id: Option<uuid::Uuid>,
generation: ReplicationGenerationSnapshot,
}
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
fn from(info: &ReplicateObjectInfo) -> Self {
Self {
bucket: info.bucket.clone(),
object: info.name.clone(),
version_id: info.version_id,
generation: info.replication_generation.clone(),
}
}
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
}
impl ReplicationInFlight {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
/// Claim `ri`; `false` when the same version is already in flight.
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
self.lock().insert(ReplicationInFlightKey::from(ri))
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
}
fn finish(&self, ri: &ReplicateObjectInfo) {
self.lock().remove(&ReplicationInFlightKey::from(ri));
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
}
#[cfg(test)]
@@ -2457,21 +2348,26 @@ impl ReplicationInFlight {
/// including when replication panics.
struct ReplicationInFlightGuard {
in_flight: Arc<ReplicationInFlight>,
key: ReplicationInFlightKey,
key: ReplicateObjectInfo,
}
impl ReplicationInFlightGuard {
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
Self {
in_flight,
key: ReplicationInFlightKey::from(ri),
key: ReplicateObjectInfo {
bucket: ri.bucket.clone(),
name: ri.name.clone(),
version_id: ri.version_id,
..Default::default()
},
}
}
}
impl Drop for ReplicationInFlightGuard {
fn drop(&mut self) {
self.in_flight.lock().remove(&self.key);
self.in_flight.finish(&self.key);
}
}
@@ -3525,8 +3421,6 @@ mod tests {
allow_first_read: Notify,
read_count: AtomicUsize,
delete_storage_calls: AtomicUsize,
delete_error: StdMutex<EcstoreError>,
replay_stamp: StdMutex<Option<Vec<u8>>>,
reported_size: StdMutex<Option<i64>>,
stream_read_bytes: Arc<AtomicUsize>,
write_count: AtomicUsize,
@@ -3594,24 +3488,6 @@ mod tests {
_h: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader, Self::Error> {
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let stamp = self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned")
.clone()
.ok_or(EcstoreError::FileNotFound)?;
return Ok(Self::GetObjectReader {
object_info: ObjectInfo {
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
..Default::default()
},
stream: Box::new(Cursor::new(stamp)),
buffered_body: None,
body_source: Default::default(),
});
}
if !object.ends_with("/.replication/resync.bin")
&& !object.ends_with("config/replication/mrf.bin")
&& !object.ends_with("config/replication/force-delete.bin")
@@ -3675,16 +3551,6 @@ mod tests {
let lock = self.new_ns_lock(bucket, object).await?;
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
};
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let mut stamp = Vec::new();
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
*self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
return Ok(ObjectInfo::default());
}
if opts.http_preconditions.is_some()
&& let Some(replacement) = self
.shared
@@ -3806,12 +3672,7 @@ mod tests {
_opts: Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error> {
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
Err(self
.shared
.delete_error
.lock()
.expect("delete error lock should not be poisoned")
.clone())
Err(EcstoreError::NotImplemented)
}
async fn delete_objects(
@@ -4068,48 +3929,6 @@ mod tests {
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
}
#[tokio::test]
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, mut rx) = mpsc::channel(4);
pool.workers.write().await.push(tx);
let initial = ReplicateObjectInfo {
bucket: "pending-tags-bucket".to_string(),
name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
size: 4096,
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..Default::default()
};
let metadata = ReplicateObjectInfo {
op_type: ReplicationType::Metadata,
user_tags: String::new(),
replication_generation: ReplicationGenerationSnapshot {
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
..Default::default()
},
..initial.clone()
};
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
assert_eq!(pool.in_flight.len(), 2);
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
assert_eq!(pool.in_flight.len(), 1);
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
for expected in [&initial, &metadata] {
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
panic!("expected object replication");
};
assert_eq!(queued.replication_generation, expected.replication_generation);
}
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
assert_eq!(pool.in_flight.len(), 0);
}
#[tokio::test]
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4352,8 +4171,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -4404,103 +4221,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_worker_already_purged_source_completes_without_retry() {
let arn = "arn:test";
for delete_marker_version in [false, true] {
for version_id in [Uuid::new_v4(), Uuid::nil()] {
for error in [
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
EcstoreError::FileVersionNotFound,
EcstoreError::FileNotFound,
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: (!delete_marker_version).then_some(version_id),
delete_marker_version_id: delete_marker_version.then_some(version_id),
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
replicate_delete_with_outcome(delete, storage).await,
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
);
assert_eq!(
shared.delete_storage_calls.load(Ordering::SeqCst),
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
"the worker must lock and attempt writeback, reading a non-null data version first"
);
}
}
}
}
#[tokio::test]
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
let arn = "arn:test";
for (version_purge, target_status, error) in [
(
true,
VersionPurgeStatusType::Pending,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
(
false,
VersionPurgeStatusType::Empty,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: version_purge.then(Uuid::new_v4),
delete_marker: !version_purge,
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
targets: if version_purge {
HashMap::new()
} else {
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
},
purge_targets: if version_purge {
HashMap::from([(arn.to_string(), target_status)])
} else {
HashMap::new()
},
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
!replicate_delete_with_outcome(delete, storage).await,
"unfinished target work, source I/O failures, and marker creation must remain retryable"
);
}
}
#[tokio::test]
async fn delete_worker_preserves_persisted_admission_without_live_config() {
let arn = "arn:test";
@@ -5584,8 +5304,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -6550,108 +6268,6 @@ mod tests {
assert_eq!(retained.target_arns, entry.target_arns);
}
#[tokio::test]
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
let shared = empty_resync_shared_state();
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
// A force-delete intent from a superseded generation replays as
// Skipped, so a pass that reaches it acknowledges it off the journal.
let stale_intent = |object: &str| MrfReplicateEntry {
bucket: "mrf-runtime-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
let journal_drained = || async {
tokio::time::timeout(Duration::from_secs(5), async {
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.is_ok()
};
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
for object in ["first", "second"] {
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
assert!(
journal_drained().await,
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
);
}
handle.abort();
}
#[tokio::test]
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
let shared = empty_resync_shared_state();
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
let interval = Duration::from_secs(1);
let persist_backlog = |object: &str| {
let entry = MrfReplicateEntry {
bucket: "mrf-node-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[entry]).expect("MRF entry should encode");
};
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
persist_backlog("first");
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
// Node B's timer fires within the same interval with a backlog present.
persist_backlog("second");
assert!(
!replay_mrf_journal(&node_b, Some(interval)).await,
"a node must not replay within one interval of another node's pass"
);
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
assert!(
replay_mrf_journal(&node_b, None).await,
"the startup pass should replay regardless of the shared stamp"
);
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
persist_backlog("third");
tokio::time::sleep(interval).await;
assert!(
replay_mrf_journal(&node_a, Some(interval)).await,
"a periodic pass should replay once the interval has elapsed"
);
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
}
#[test]
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
let interval = Duration::from_secs(300);
let now = 1_000_000_000;
assert!(mrf_replay_due(None, now, interval));
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
assert!(
!mrf_replay_due(Some(now + 1_000), now, interval),
"small clock skew should still space passes"
);
assert!(
mrf_replay_due(Some(now + 300_000), now, interval),
"a stamp an interval ahead must not suppress replay"
);
}
#[test]
fn mrf_entry_delete_marker_roundtrip() {
let dm_vid = Uuid::new_v4();
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
true
}
Err(e) => {
// A batch purge or another worker may have removed this version
// already. Remote failures still control the final outcome below.
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
if !source_already_purged {
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
}
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
},
..Default::default()
});
source_already_purged
false
}
};
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
/// If the marker disappears (deleted before or while the replica landed),
/// purge the replicated marker from the targets, retrying failed targets on
/// later rounds. When the window drains with targets still dirty, persist the
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
/// purge intent as a durable MRF entry so the next startup replays it through
/// `purge_stale_delete_marker_targets`.
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
/// Heal callers retry for the full watch window because the startup MRF
/// processor runs before bucket metadata (and thus target clients) finishes
/// initializing — the first pass can see `target_client_missing` and a later
/// round resolves the client; replay passes are serial background work, so
/// round resolves the client; the replay loop is serial and startup-only, so
/// blocking it for up to the window per dirty entry is acceptable. Live
/// callers run on replication workers where a down target would pin a worker
/// for the whole window, so they attempt once and lean on the durable intent
@@ -6609,7 +6604,6 @@ mod tests {
health_check_duration: std::time::Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
timeouts: Default::default(),
})
}
-12
View File
@@ -259,10 +259,6 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
check_put_object_args(bucket, object)?;
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
}
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
return Err(StorageError::InvalidArgument(
bucket.to_owned(),
@@ -274,14 +270,6 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
Ok(())
}
/// New public keys must remain representable in an unencoded XML listing.
/// Existing keys still use the read/delete validators so they can be removed.
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
object
.chars()
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
}
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
check_bucket_and_object_names(bucket, object)
}
+33 -238
View File
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
use tokio::task::JoinSet;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{Instrument, debug, error, warn};
use tracing::{debug, error, warn};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -224,8 +224,6 @@ pub struct ListPathRawOptions {
pub recursive: bool,
pub incl_deleted: bool,
pub skip_hidden_prefix_check: bool,
pub hidden_entry_budget: usize,
pub pending_prefix: Option<String>,
pub filter_prefix: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
@@ -262,8 +260,6 @@ impl Clone for ListPathRawOptions {
recursive: self.recursive,
incl_deleted: self.incl_deleted,
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
hidden_entry_budget: self.hidden_entry_budget,
pending_prefix: self.pending_prefix.clone(),
filter_prefix: self.filter_prefix.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
@@ -293,8 +289,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
recursive: opts.recursive,
incl_deleted: opts.incl_deleted,
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
hidden_entry_budget: opts.hidden_entry_budget,
pending_prefix: opts.pending_prefix.clone(),
report_notfound: opts.report_not_found,
filter_prefix: opts.filter_prefix.clone(),
forward_to: opts.forward_to.clone(),
@@ -307,15 +301,13 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
}
/// How a merged raw listing stream ended.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ListPathRawOutcome {
/// At least one producer stopped at its per-disk limit instead of at the
/// end of its namespace, so the closed stream is not proof of exhaustion.
/// Carried in-band by the metacache stream trailer, so remote producers
/// report it as well.
pub limit_reached: bool,
/// Inclusive sorted boundary proved by every reader before a bounded stop.
pub scan_cut: Option<String>,
}
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -371,15 +363,10 @@ async fn list_path_raw_inner(
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
let cancel_rx = CancellationToken::new();
let walk_options = Arc::new(walk_dir_options(&opts));
for (disk_idx, disk) in opts.disks.iter().enumerate() {
let opdisk = disk.clone();
let walk_options = walk_options.clone();
#[cfg(test)]
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
#[cfg(test)]
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
let opts_clone = opts.clone();
let fallback_claim_tracker = fallback_claim_tracker.clone();
let fds_clone = fds.clone();
#[cfg(test)]
@@ -389,17 +376,17 @@ async fn list_path_raw_inner(
let disk_io_context_clone = disk_io_context.clone();
let (rd, wr) = tokio::io::duplex(64 * 1024);
readers.push(MetacacheReader::new(rd));
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
let mut wr = PublishedBytesWriter::new(wr);
#[cfg(test)]
if let Some(observer) = &test_disk_io_context_observed {
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
observer.store(
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
std::sync::atomic::Ordering::Release,
);
}
#[cfg(test)]
let test_primary_error = if let Some(behavior) = test_reader_behavior {
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
match behavior {
TestReaderBehavior::Eof => return Ok(()),
TestReaderBehavior::Entries(entries) => {
@@ -456,6 +443,8 @@ async fn list_path_raw_inner(
None
};
let wakl_opts = walk_dir_options(&opts_clone);
let mut need_fallback = false;
let mut last_err = None;
#[cfg(test)]
@@ -470,7 +459,7 @@ async fn list_path_raw_inner(
if !need_fallback && let Some(disk) = opdisk {
let primary_walk_started = std::time::Instant::now();
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
match disk.walk_dir(wakl_opts, &mut wr).await {
Ok(_res) => {
rustfs_io_metrics::record_stage_duration(
"metacache_walk_dir_primary",
@@ -494,8 +483,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_missing_path",
error = ?err,
@@ -506,8 +495,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "walk_dir_failed",
error = ?err,
@@ -609,8 +598,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing_for_path",
error = ?err,
@@ -621,8 +610,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_disk_missing",
error = ?err,
@@ -637,7 +626,7 @@ async fn list_path_raw_inner(
}
let fallback_walk_started = std::time::Instant::now();
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
Ok(_r) => {
rustfs_io_metrics::record_stage_duration(
"metacache_walk_dir_fallback",
@@ -663,8 +652,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_missing_path",
error = ?err,
@@ -675,8 +664,8 @@ async fn list_path_raw_inner(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
bucket = %walk_options.bucket,
path = %walk_options.base_dir,
bucket = %opts_clone.bucket,
path = %opts_clone.path,
disk_index = disk_idx,
state = "fallback_walk_dir_failed",
error = ?err,
@@ -698,13 +687,12 @@ async fn list_path_raw_inner(
// warn!("list_path_raw: while need_fallback done");
Ok(())
});
jobs.spawn(producer.in_current_span());
}));
}
let revjob_rx = rx.clone();
let mut revjobs = JoinSet::new();
let merge = async move {
revjobs.spawn(async move {
// Consumer-side peek timeout, and a caveat worth understanding
// (rustfs/backlog#1217).
//
@@ -734,8 +722,6 @@ async fn list_path_raw_inner(
// Sticky across rounds: once any reader's trailer says its producer
// stopped at the per-disk limit, the merged stream may hide more keys.
let mut limit_reached = false;
let mut scan_cut = None;
let mut last_scanned_key: Option<String> = None;
loop {
let mut current = MetaCacheEntry::default();
@@ -747,7 +733,7 @@ async fn list_path_raw_inner(
// );
if revjob_rx.is_cancelled() {
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
return Ok(ListPathRawOutcome { limit_reached });
}
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -807,12 +793,7 @@ async fn list_path_raw_inner(
continue;
}
// eof
if r.limit_reached() == Some(true) {
limit_reached = true;
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
DiskError::other("bounded listing reader ended without a scan boundary")
})?);
}
limit_reached |= r.limit_reached() == Some(true);
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
@@ -886,7 +867,7 @@ async fn list_path_raw_inner(
*r = MetacacheReader::new(detached_rd);
continue;
}
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
}
};
@@ -945,7 +926,7 @@ async fn list_path_raw_inner(
if let Some(finished_fn) = opts.finished.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -995,14 +976,6 @@ async fn list_path_raw_inner(
return Err(classify_listing_quorum_failure(&failures));
}
// The next head from a bounded reader is unknown. The previous
// globally consumed group is a safe cut; consuming another reader's
// later head would turn this absence of evidence into a skipped key.
// Check quorum failures above before accepting the bounded result.
if limit_reached {
break;
}
// Break if all at EOF or error.
if at_eof + has_err == readers.len() {
if has_err > 0
@@ -1010,7 +983,7 @@ async fn list_path_raw_inner(
{
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = finished_fn(&errs) => {}
}
}
@@ -1021,7 +994,6 @@ async fn list_path_raw_inner(
break;
}
last_scanned_key = Some(current.name.clone());
if agree == readers.len() && !opts.preserve_replica_metadata {
for r in readers.iter_mut() {
if !r.skip_current() {
@@ -1033,7 +1005,7 @@ async fn list_path_raw_inner(
// warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
result = agreed_fn(current) => result?
}
// warn!("list_path_raw: agreed_fn done");
@@ -1057,14 +1029,13 @@ async fn list_path_raw_inner(
if let Some(partial_fn) = opts.partial.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
}
}
}
Ok(ListPathRawOutcome { limit_reached, scan_cut })
};
revjobs.spawn(merge.in_current_span());
Ok(ListPathRawOutcome { limit_reached })
});
let merge_started = std::time::Instant::now();
let Some(revjob) = revjobs.join_next().await else {
@@ -1228,24 +1199,6 @@ mod tests {
assert!(!options.skip_total_timeout);
}
#[test]
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
let request = ListPathRawOptions {
bucket: "versioned".to_owned(),
hidden_entry_budget: 100,
pending_prefix: Some("pending/".to_owned()),
forward_to: Some("pending/0099".to_owned()),
..Default::default()
};
// Primary and fallback readers both construct their walk from the
// per-job clone, rather than directly from the request options.
let job = request.clone();
let walk = walk_dir_options(&job);
assert_eq!(walk.hidden_entry_budget, 100);
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
}
#[tokio::test]
async fn list_path_raw_empty_disks_returns_read_quorum() {
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1296,163 +1249,6 @@ mod tests {
);
}
#[tokio::test]
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
let root = tempfile::tempdir().expect("create disk root");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
.expect("parse disk endpoint");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("open local disk"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
let baseline = Arc::strong_count(&disk);
for primary_count in [1, 4, 16] {
for fallback_count in [0, 2] {
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
let behaviors = entered
.iter()
.zip(&stopped)
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
entered: entered.clone(),
stopped: stopped.clone(),
})
.collect();
let cancel = CancellationToken::new();
let listing = tokio::spawn(list_path_raw(
cancel.clone(),
ListPathRawOptions {
disks: vec![Some(disk.clone()); primary_count],
fallback_disks: vec![Some(disk.clone()); fallback_count],
min_disks: primary_count,
test_reader_behaviors: behaviors,
peek_timeout: Some(Duration::from_secs(60)),
..Default::default()
},
));
for signal in &entered {
timeout(Duration::from_secs(2), signal.notified())
.await
.expect("every producer must enter the real fanout before counting references");
}
// The merge owns the configured primary/fallback slots, each producer
// owns its primary, and the shared fallback queue owns each fallback.
let expected = baseline + 2 * (primary_count + fallback_count);
let retained = Arc::strong_count(&disk);
cancel.cancel();
timeout(Duration::from_secs(2), listing)
.await
.expect("cancellation must stop the listing")
.expect("listing task must join")
.expect("external cancellation must finish cleanly");
for signal in &stopped {
timeout(Duration::from_secs(2), signal.notified())
.await
.expect("cancellation must drop every producer");
}
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
assert_eq!(
retained, expected,
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
);
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn list_path_raw_producer_and_merge_preserve_request_span() {
use tracing_subscriber::{Layer, layer::SubscriberExt};
struct ProducerSpan(Arc<AtomicBool>);
#[derive(Default)]
struct StateField(bool);
impl tracing::field::Visit for StateField {
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
}
}
impl<S> Layer<S> for ProducerSpan
where
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
{
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
let mut state = StateField::default();
event.record(&mut state);
if state.0
&& context
.lookup_current()
.is_some_and(|span| span.name() == "flink_list_request")
{
self.0.store(true, std::sync::atomic::Ordering::Release);
}
}
}
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
let producer_observed = Arc::new(AtomicBool::new(false));
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
let _subscriber = tracing::subscriber::set_default(subscriber);
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
let root = tempfile::tempdir().expect("create disk root");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
.expect("parse disk endpoint");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("open local disk"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
let error = list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![Some(disk)],
bucket: "missing-bucket".to_owned(),
min_disks: 1,
..Default::default()
},
)
.instrument(request.clone())
.await
.expect_err("missing bucket must fail the real disk producer");
assert_eq!(error, DiskError::VolumeNotFound);
assert!(
producer_observed.load(std::sync::atomic::Ordering::Acquire),
"disk error must retain the request span"
);
let merge_observed = Arc::new(AtomicBool::new(false));
let merge_sink = merge_observed.clone();
let request_id = request.id();
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
agreed: Some(Box::new(move |_| {
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
Box::pin(async { Ok(()) })
})),
..Default::default()
},
)
.instrument(request)
.await
.expect("merge should deliver the test entry");
assert!(
merge_observed.load(std::sync::atomic::Ordering::Acquire),
"merge callback must retain the request span"
);
}
#[tokio::test]
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
let root = tempfile::tempdir().expect("local disk root should be created");
@@ -1520,7 +1316,6 @@ mod tests {
outcome.limit_reached,
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
);
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
}
#[tokio::test]
+4 -45
View File
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
#[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
global_internode_metrics,
};
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
use rustfs_utils::get_env_bool;
use sha2::Digest as _;
use sha2::Sha256;
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
use tracing::{error, info, warn};
use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -1588,13 +1590,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_tonic_nonce(nonce, signed_at, path)?;
// A body-bound request also carries a usable v2 signature. Consume both
// nonces so stripping the replay-scope headers cannot authorize it again.
if content_sha256 != UNSIGNED_PAYLOAD {
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
}
Ok(())
check_and_record_tonic_nonce(nonce, signed_at, path)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -2585,43 +2581,6 @@ mod tests {
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_consumes_the_body_bound_v2_nonce() {
ensure_test_rpc_secret();
let audience = "node-a:9000";
let path = "/node_service.NodeService/RecoverDeleteTransaction";
let digest = "a".repeat(64);
for v3_first in [true, false] {
let mut headers =
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
.expect("body-bound v2 headers should build");
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
let mut downgraded = headers.clone();
for header in [
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
] {
downgraded.remove(header);
}
let (first, replay) = if v3_first {
(&headers, &downgraded)
} else {
(&downgraded, &headers)
};
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
.expect("the first delivery must be accepted");
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
.expect_err("a mutation cannot replay through the other signature generation");
assert_eq!(error.to_string(), "RPC request replay detected");
}
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
@@ -2014,21 +2014,9 @@ impl PeerRestClient {
}
result
})
.await;
self.settle_bucket_metadata_reload(bucket, result).await
}
async fn settle_bucket_metadata_reload(
&self,
bucket: &str,
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
) -> Result<()> {
match result {
Ok(result) => self.finalize_result(result).await,
// A local operation deadline does not establish that the peer is
// unreachable. In particular, it must not gate IAM revocations.
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
}
.await
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
self.finalize_result(result).await
}
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4765,31 +4753,6 @@ mod tests {
assert!(client.offline.load(Ordering::Acquire));
}
#[tokio::test]
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
let client = test_peer_client();
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
let error = client
.settle_bucket_metadata_reload("slow-policy", elapsed)
.await
.expect_err("local budget expires");
assert!(error.to_string().contains("timed out"));
assert!(
!client.offline.load(Ordering::Acquire),
"local work must not suppress subsequent IAM RPCs"
);
client
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
.await
.expect("peer reply");
let error = client
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
.await
.expect_err("real transport failure");
assert!(error.to_string().contains("connection refused"));
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
}
#[tokio::test]
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
let client = test_peer_client();
+1 -310
View File
@@ -1082,10 +1082,7 @@ impl RemoteDisk {
if payload.len() > operation.max_wire_payload_bytes() {
return Err(Error::other("delete recovery operation exceeds payload limit"));
}
let settle_timeout = if matches!(
&operation,
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
) {
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
Some(crate::disk::get_delete_recovery_timeout()?)
} else {
None
@@ -1640,7 +1637,6 @@ impl RemoteDisk {
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1712,7 +1708,6 @@ impl RemoteDisk {
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1783,7 +1778,6 @@ impl RemoteDisk {
}
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -2015,7 +2009,6 @@ impl RemoteDisk {
let operation_result = operation().await;
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2030,7 +2023,6 @@ impl RemoteDisk {
// Log success; the waiting guard balances every exit path.
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2719,54 +2711,6 @@ impl RemoteDisk {
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
let expected_disk = self
.id
.lock()
.await
.filter(|id| !id.is_nil())
.ok_or(DiskError::DiskNotFound)?;
let query = MultipartIndexRpcRequest::new(bucket, marker);
let endpoint = self.endpoint.to_string();
let canonical = query.canonical_request(&endpoint)?;
let opts = serde_json::to_string(&query)?;
self.execute_with_timeout(
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(DiskInfoRequest {
disk: endpoint.clone(),
opts: opts.clone(),
});
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
let response = client.disk_info(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
return Err(DiskError::FileCorrupt);
}
// An old server ignores the additive query and returns ordinary DiskInfo.
// It cannot satisfy the dedicated reply schema or the fresh challenge.
let reply: MultipartIndexRpcReply =
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
reply.validate(&query, expected_disk)?;
crate::cluster::rpc::verify_tonic_rpc_response_proof(
&query.canonical_response(&endpoint, &reply.page)?,
&reply.proof,
)
.map_err(Error::other)?;
Ok(reply.page)
},
get_drive_disk_info_timeout(),
)
.await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4350,38 +4294,6 @@ impl DiskAPI for RemoteDisk {
}
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
// Older peers reject this variant. Do not retry through the unbounded protocol;
// they are reported as unsupported participants (rolling-upgrade contract).
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::DiscoverBounded {
object: object.to_owned(),
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -4417,62 +4329,6 @@ impl DiskAPI for RemoteDisk {
_ => Err(Error::other("unexpected delete discovery reply")),
}
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
_namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
match self
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
.await
{
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
match self
.recover_delete_transaction(
volume,
DeleteRecoveryRpcOperation::InspectBounded {
object: object.to_owned(),
transaction,
max_entries,
},
)
.await
{
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
Ok(_) => Err(Error::FileCorrupt),
Err(error)
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
{
// No paginated fallback through the unbounded protocol; the peer
// is an unsupported participant until it is upgraded.
Err(Error::UnsupportedDisk)
}
Err(error) => Err(error),
}
}
async fn inspect_delete_transaction(
&self,
volume: &str,
@@ -4993,113 +4849,6 @@ mod tests {
}
}
#[tokio::test]
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("test peer must bind");
let directory = tempfile::tempdir().expect("temporary disk");
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk"),
);
disk.make_volume("bucket").await.expect("bucket");
let prefix = directory.path().join("bucket/prefix");
std::fs::create_dir(&prefix).expect("prefix");
for index in 0..64 {
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
}
let fixture = Arc::new(TestBoundedDiscovery {
disk: Arc::clone(&disk),
response_gate: TestFormatReadGate::default(),
});
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
let remote = remote_disk_for_addr(&peer.addr).await;
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
.await
.expect("disk completed bounded scan");
task.abort();
assert!(task.await.expect_err("caller cancelled").is_cancelled());
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
fixture.response_gate.release.cancel();
peer.stop().await;
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
assert!(prefix.join("child-63").is_dir());
}
#[tokio::test]
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
remote
.set_disk_id(Some(Uuid::new_v4()))
.await
.expect("expected disk identity");
assert!(matches!(
remote.list_multipart_index_roots("bucket", None).await,
Err(DiskError::MethodNotAllowed)
));
assert_eq!(
peer.peer.disk_info_calls.load(Ordering::Acquire),
1,
"exercise the actual legacy JSON handler"
);
peer.stop().await;
}
#[tokio::test]
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
assert_eq!(
remote
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
.await
.expect_err("old peer cannot prove a bounded inventory"),
DiskError::UnsupportedDisk
);
let transaction = Uuid::new_v4();
assert_eq!(
remote
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
.await
.expect_err("old peer cannot inspect with a budget"),
DiskError::UnsupportedDisk
);
assert_eq!(
remote
.settle_delete_transaction_bounded(
"bucket",
crate::disk::BoundedDeleteRecoverySettlement {
object: "prefix".to_owned(),
transaction,
expected_fingerprint: [0; 32],
action: DeleteRecoveryAction::PublishCommit,
max_entries: 65536,
},
None
)
.await
.expect_err("old peer cannot settle with a budget"),
DiskError::UnsupportedDisk
);
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods"),
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
);
peer.stop().await;
}
#[tokio::test]
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -5992,16 +5741,9 @@ mod tests {
format_data: Bytes,
read_all_data: Bytes,
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
unsupported_methods: Arc<StdMutex<Vec<String>>>,
}
#[derive(Debug)]
struct TestBoundedDiscovery {
disk: Arc<crate::disk::local::LocalDisk>,
response_gate: TestFormatReadGate,
}
#[derive(Debug, Default)]
struct TestFormatReadGate {
entered: tokio::sync::Notify,
@@ -6018,7 +5760,6 @@ mod tests {
format_data,
read_all_data,
next_format_read_gate: Arc::default(),
bounded_discovery: Arc::default(),
unsupported_methods: Arc::default(),
}
}
@@ -6118,56 +5859,6 @@ mod tests {
Ok(grpc.unary(method, request).await)
})
}
"/node_service.NodeService/RecoverDeleteTransaction"
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
{
#[derive(Clone)]
struct BoundedSvc(Arc<TestBoundedDiscovery>);
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
let fixture = Arc::clone(&self.0);
Box::pin(async move {
let request = request.into_inner();
let operation: DeleteRecoveryRpcOperation =
rustfs_protos::decode_delete_recovery_payload(&request.operation)
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
return Err(Status::invalid_argument("unexpected fallback"));
};
let page = fixture
.disk
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
.await
.map_err(|_| Status::internal("bounded discovery failed"))?;
// Lose the response only after the real disk operation has completed.
fixture.response_gate.entered.notify_one();
fixture.response_gate.release.cancelled().await;
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
.map_err(|_| Status::internal("encode bounded reply"))?;
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
success: true,
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
result: result.into(),
error: None,
}))
})
}
}
let fixture = self
.peer
.bounded_discovery
.lock()
.expect("bounded fixture")
.clone()
.expect("configured fixture");
Box::pin(async move {
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec);
Ok(grpc.unary(BoundedSvc(fixture), request).await)
})
}
"/node_service.NodeService/ReadAll" => {
#[derive(Clone)]
struct ReadAllSvc(AuthenticatedReadPeer);
+56 -532
View File
@@ -18,7 +18,6 @@ use crate::cluster::rpc::client::{
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
use async_trait::async_trait;
use bytes::Bytes;
use futures::{FutureExt, future::BoxFuture};
use rustfs_lock::{
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
types::{LockId, LockMetadata, LockPriority},
@@ -33,17 +32,16 @@ use rustfs_protos::{
};
use std::collections::HashMap;
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::pin::Pin;
use std::sync::{Mutex, OnceLock};
use std::time::Duration;
use tokio::task::JoinHandle;
use tokio::time::{Instant, timeout};
use tonic::service::interceptor::InterceptedService;
use tonic::{Request, Response};
use tracing::{debug, info, warn};
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
@@ -51,38 +49,18 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
set_tonic_rolling_mutation_body_digest(request)
}
/// Work to run if an RPC completes after its caller stops waiting.
/// Work to run if an RPC that already timed out for its caller completes later.
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
type LateFailure = LateCompletion<()>;
type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
/// Own admission before connecting as well as while an RPC is in flight.
struct LockRequestGuard<'a> {
client: &'a RemoteClient,
op: &'static str,
struct RpcCleanup<T> {
late: LateCompletion<T>,
late_failure: LateFailure,
admission: Option<LockRequestAdmission>,
}
impl LockRequestGuard<'_> {
fn into_admission(mut self) -> Option<LockRequestAdmission> {
self.admission.take()
}
}
impl Drop for LockRequestGuard<'_> {
fn drop(&mut self) {
self.client.release_lock_request(self.op, self.admission.take());
}
}
struct RpcCleanup<'a, T> {
late: LateCompletion<T>,
late_failure: LateFailure,
admission: Option<LockRequestGuard<'a>>,
}
impl<'a, T> RpcCleanup<'a, T> {
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
impl<T> RpcCleanup<T> {
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
Self {
late,
late_failure,
@@ -91,25 +69,6 @@ impl<'a, T> RpcCleanup<'a, T> {
}
}
/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
struct InFlightRpc<'a, T: Send + 'static> {
client: &'a RemoteClient,
op: &'static str,
resource_summary: &'a str,
future: Option<RpcFuture<T>>,
cleanup: Option<RpcCleanup<'a, T>>,
}
impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
fn drop(&mut self) {
if let Some(future) = self.future.take()
&& let Some(cleanup) = self.cleanup.take()
{
self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
}
}
}
/// The liveness window is this many RPC deadlines: a peer that completed a
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
@@ -178,9 +137,6 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
let mut peers = lock_peer_channel_health()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(health) = peers.get_mut(addr) {
return update(health);
}
update(peers.entry(addr.to_string()).or_default())
}
@@ -202,15 +158,6 @@ fn reset_lock_peer_health_for_test(addr: &str) {
.remove(addr);
}
#[cfg(test)]
type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
#[cfg(test)]
fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
PAUSES.get_or_init(Mutex::default)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EvictionTrigger {
/// The caller's deadline expired while the stream was still open.
@@ -498,10 +445,6 @@ impl RemoteClient {
}
}
});
// A successful lock RPC over a cached or fresh channel proves the peer is
// reachable; reconcile the internode peer-health gauge so it does not stay
// stale when no fresh dial fires (issue #9119).
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
fn admit_late_release(&self) -> bool {
@@ -559,14 +502,6 @@ impl RemoteClient {
admission
}
fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
self.admit_lock_request(op).map(|admission| LockRequestGuard {
client: self,
op,
admission: Some(admission),
})
}
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
if is_request_breaker_operation(op) && admission.is_some() {
with_lock_peer_health(&self.addr, |health| {
@@ -648,17 +583,6 @@ impl RemoteClient {
trigger: EvictionTrigger,
deadline: Duration,
) {
#[cfg(test)]
let pause = eviction_pauses_for_test()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&self.addr);
#[cfg(test)]
if let Some((entered, resume)) = pause {
let _ = entered.send(());
let _ = resume.await;
}
let now = Instant::now();
let cooldown = Self::eviction_cooldown();
let liveness_window = Self::liveness_window(deadline);
@@ -689,26 +613,22 @@ impl RemoteClient {
);
}
/// Keep an abandoned RPC running instead of cancelling its stream.
/// Keep a timed-out RPC running instead of cancelling its stream.
///
/// Dropping the future sends `RST_STREAM`; under load those resets pile up
/// in the server's pending-accept queue until it answers `GOAWAY
/// too_many_resets` and kills every stream on the connection. A detached
/// stream ends on its own within the internode RPC timeout, the number per
/// peer is bounded, and a lock granted after its caller gave up is released.
fn detach_rpc<T: Send + 'static>(
fn detach_timed_out_rpc<T: Send + 'static>(
&self,
op: &'static str,
resource_summary: &str,
future: RpcFuture<T>,
cleanup: RpcCleanup<'_, T>,
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
late: LateCompletion<T>,
late_failure: LateFailure,
admission: Option<LockRequestAdmission>,
) {
let RpcCleanup {
late,
late_failure,
admission,
} = cleanup;
let admission = admission.and_then(LockRequestGuard::into_admission);
let limit = Self::detached_rpc_limit();
let admitted = with_lock_peer_health(&self.addr, |health| {
if health.detached_rpcs >= limit {
@@ -719,29 +639,25 @@ impl RemoteClient {
}
});
if !admitted {
drop(future);
handle.abort();
self.release_lock_request(op, admission);
if let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "detached_budget_exhausted",
addr = %self.addr,
op,
resource_summary,
limit,
"Abandoned remote lock RPC cancelled"
"Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
);
return;
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
let addr = self.addr.clone();
tokio::spawn(async move {
let outcome = future.await;
let outcome = handle.await;
with_lock_peer_health(&addr, |health| {
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
if is_request_breaker_operation(op) && admission.is_some() {
@@ -756,7 +672,6 @@ impl RemoteClient {
with_lock_peer_health(&addr, |health| {
health.last_success = Some(Instant::now());
});
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
if let Some(late) = late {
late(response).await;
@@ -768,31 +683,19 @@ impl RemoteClient {
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_rpc_failed",
addr = %addr,
op,
tonic_code = ?status.code(),
tonic_message = status.message(),
"Abandoned remote lock RPC failed"
"Detached remote lock RPC failed after its caller timed out"
);
}
Err(_) => {
Err(join_error) => {
if let Some(late_failure) = late_failure {
late_failure(()).await;
}
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_rpc_panicked",
addr = %addr,
op,
"Abandoned remote lock RPC panicked"
);
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
}
}
});
@@ -880,28 +783,16 @@ impl RemoteClient {
self.release_late_release();
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
if outcome == "released" {
debug!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_grants_released",
addr = %self.addr,
count = original_count,
"Released remote locks granted after their caller stopped waiting"
);
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
} else {
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
if log_warning {
warn!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "late_grants_unreleased",
addr = %self.addr,
count = original_count,
outcome,
suppressed_timeouts,
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
);
} else {
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
@@ -915,47 +806,25 @@ impl RemoteClient {
resource_summary: &str,
deadline: Duration,
future: Fut,
cleanup: RpcCleanup<'_, T>,
cleanup: RpcCleanup<T>,
) -> std::result::Result<T, LockError>
where
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
T: Send + 'static,
{
let mut rpc = InFlightRpc {
client: self,
op,
resource_summary,
future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
cleanup: Some(cleanup),
};
let outcome = match rpc.future.as_mut() {
Some(future) => tokio::select! {
// A response becoming ready on the deadline tick must not turn an
// expired caller budget into a transport error and channel eviction.
biased;
_ = tokio::time::sleep(deadline) => None,
outcome = future => Some(outcome),
},
None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
};
if outcome.is_some() {
rpc.future.take();
}
let Some(RpcCleanup {
late: _,
let RpcCleanup {
late,
late_failure,
admission,
}) = rpc.cleanup.as_mut()
else {
return Err(LockError::internal("remote lock RPC has no cleanup owner"));
};
let token = admission.as_ref().and_then(|guard| guard.admission);
let result = match outcome {
Some(Ok(Ok(response))) => {
self.record_rpc_success(op, token);
} = cleanup;
let mut handle = tokio::spawn(future);
match timeout(deadline, &mut handle).await {
Ok(Ok(Ok(response))) => {
self.record_rpc_success(op, admission);
self.release_lock_request(op, admission);
Ok(response)
}
Some(Ok(Err(err))) => {
Ok(Ok(Err(err))) => {
let reason = err.to_string();
if err.code() == tonic::Code::DeadlineExceeded {
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
@@ -988,11 +857,12 @@ impl RemoteClient {
"Suppressed repeated remote lock RPC deadline"
);
}
if let Some(late_failure) = late_failure.take() {
drop(tokio::spawn(late_failure(())));
}
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
.await;
self.release_lock_request(op, admission);
if let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
}
// Only evict (and re-dial) the cached channel when the failure is a genuine
@@ -1053,32 +923,30 @@ impl RemoteClient {
);
}
if transport_failure {
// Ambiguous-grant cleanup owns its work before channel eviction can suspend.
if let Some(late_failure) = late_failure.take() {
drop(tokio::spawn(late_failure(())));
}
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
.await;
}
self.release_lock_request(op, admission);
if transport_failure && let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
}
Some(Err(_)) => {
Ok(Err(join_error)) => {
warn!(
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "rpc_panicked",
addr = %self.addr,
op,
resource_summary,
"Remote lock RPC panicked"
error = %join_error,
"Remote lock RPC task ended abnormally"
);
if let Some(late_failure) = late_failure.take() {
self.release_lock_request(op, admission);
if let Some(late_failure) = late_failure {
drop(tokio::spawn(late_failure(())));
}
Err(LockError::internal(format!("{op} RPC future panicked")))
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
}
None => {
Err(_) => {
let reason = format!("RPC timed out after {deadline:?}");
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
self.record_request_timeout(op, deadline);
@@ -1110,17 +978,12 @@ impl RemoteClient {
"Suppressed repeated remote lock RPC timeout"
);
}
// Continue draining the abandoned stream while channel eviction waits.
drop(rpc);
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
.await;
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
}
};
if rpc.future.is_none() {
rpc.cleanup.take();
}
result
}
fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
@@ -1211,7 +1074,7 @@ impl RemoteClient {
#[async_trait]
impl LockClient for RemoteClient {
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
let admission = match self.admit_lock_request_guard("lock") {
let admission = match self.admit_lock_request("lock") {
Some(admission) => admission,
None => return Ok(Self::rpc_backoff_failure_response(request)),
};
@@ -1222,6 +1085,7 @@ impl LockClient for RemoteClient {
Ok(client) => client,
Err(err) => {
self.record_request_timeout("lock", rpc_timeout);
self.release_lock_request("lock", Some(admission));
return Err(err);
}
};
@@ -1229,12 +1093,14 @@ impl LockClient for RemoteClient {
let args = match serde_json::to_string(&bounded_request) {
Ok(args) => args,
Err(err) => {
self.release_lock_request("lock", Some(admission));
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
}
};
let mut req = Request::new(GenerallyLockRequest { args });
req.set_timeout(rpc_timeout);
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
self.release_lock_request("lock", Some(admission));
return Err(err.into());
}
let late = self.late_release_hook(request.lock_id.clone());
@@ -1276,7 +1142,7 @@ impl LockClient for RemoteClient {
return Ok(Vec::new());
}
let admission = match self.admit_lock_request_guard("lock_batch") {
let admission = match self.admit_lock_request("lock_batch") {
Some(admission) => admission,
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
};
@@ -1290,6 +1156,7 @@ impl LockClient for RemoteClient {
Ok(client) => client,
Err(err) => {
self.record_request_timeout("lock_batch", rpc_timeout);
self.release_lock_request("lock_batch", Some(admission));
return Err(err);
}
};
@@ -1303,12 +1170,14 @@ impl LockClient for RemoteClient {
{
Ok(args) => args,
Err(err) => {
self.release_lock_request("lock_batch", Some(admission));
return Err(err);
}
};
let mut req = Request::new(BatchGenerallyLockRequest { args });
req.set_timeout(rpc_timeout);
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
self.release_lock_request("lock_batch", Some(admission));
return Err(err.into());
}
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
@@ -1836,351 +1705,6 @@ mod tests {
assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
}
#[tokio::test]
async fn test_remote_client_cancelled_connection_releases_request_admission() {
ensure_test_rpc_secret();
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
let client = RemoteClient::new(addr.clone());
let request = test_lock_request(Duration::from_secs(30));
for batch in [false, true] {
reset_lock_peer_health_for_test(&addr);
with_lock_peer_health(&addr, |health| {
health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
});
let mut acquisition = Box::pin(async {
if batch {
client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
} else {
client.acquire_lock(&request).await.map(|_| ())
}
});
std::future::poll_fn(|cx| {
assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
std::task::Poll::Ready(())
})
.await;
let pending = lock_peer_health_for_test(&addr);
assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
drop(acquisition);
let cancelled = lock_peer_health_for_test(&addr);
assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
client.release_lock_request("lock", Some(probe));
}
reset_lock_peer_health_for_test(&addr);
}
#[tokio::test]
async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
let addr = "http://cancelled-rpc-late-grant";
reset_lock_peer_health_for_test(addr);
let client = RemoteClient::new(addr.to_string());
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
Box::pin(async move {
released_tx.send(granted).expect("late grant observer remains alive");
})
}));
client
.execute_rpc(
"lock",
"bucket/object",
Duration::from_secs(30),
async move {
started_tx.send(()).expect("caller observes RPC dispatch");
Ok(grant_rx.await.expect("test decides when peer grants lock"))
},
RpcCleanup::new(late, None, None),
)
.await
});
timeout(Duration::from_secs(1), started_rx)
.await
.expect("RPC must dispatch")
.expect("dispatch signal");
caller.abort();
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
1,
"cancelled RPC uses bounded detached ownership"
);
grant_tx.send(true).expect("cancellation must preserve the running RPC");
assert!(
timeout(Duration::from_secs(1), released_rx)
.await
.expect("late grant must be cleaned up")
.expect("cleanup result")
);
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
let addr = "http://cancelled-rpc-late-failure";
reset_lock_peer_health_for_test(addr);
let client = RemoteClient::new(addr.to_string());
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx.send(()).expect("ambiguous cleanup observer remains alive");
})
}));
client
.execute_rpc(
"lock_batch",
"bucket/object",
Duration::from_secs(30),
async move {
started_tx.send(()).expect("caller observes RPC dispatch");
failure_rx.await.expect("test decides when transport fails");
Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
},
RpcCleanup::new(None, late_failure, None),
)
.await
});
timeout(Duration::from_secs(1), started_rx)
.await
.expect("RPC must dispatch")
.expect("dispatch signal");
caller.abort();
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
failure_tx.send(()).expect("cancellation must preserve the running RPC");
timeout(Duration::from_secs(1), released_rx)
.await
.expect("ambiguous grant must be cleaned up")
.expect("cleanup signal");
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
for (label, status) in [
("transport", tonic::Status::cancelled("ambiguous transport failure")),
("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
] {
let addr = format!("http://cancelled-completed-{label}");
reset_lock_peer_health_for_test(&addr);
let client = RemoteClient::new(addr.clone());
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
eviction_pauses_for_test()
.lock()
.expect("eviction pause registry")
.insert(addr.clone(), (evicting_tx, resume_rx));
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx
.send(())
.expect("completed-failure cleanup observer remains alive");
})
}));
client
.execute_rpc(
"lock",
"bucket/object",
Duration::from_secs(30),
async move { Err::<(), _>(status) },
RpcCleanup::new(None, late_failure, None),
)
.await
});
timeout(Duration::from_secs(1), evicting_rx)
.await
.expect("completed error must reach channel eviction")
.expect("eviction signal");
timeout(Duration::from_secs(1), released_rx)
.await
.expect("cleanup must transfer before channel eviction suspends")
.expect("cleanup signal");
caller.abort();
assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
assert_eq!(
lock_peer_health_for_test(&addr).detached_rpcs,
0,
"completed RPC cannot consume a detached slot"
);
reset_lock_peer_health_for_test(&addr);
}
}
#[tokio::test(start_paused = true)]
async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
let addr = "http://same-tick-rpc-deadline";
reset_lock_peer_health_for_test(addr);
with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
let client = RemoteClient::new(addr.to_string());
let budget = Duration::from_millis(50);
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx.send(()).expect("same-tick cleanup observer remains alive");
})
}));
let result = client
.execute_rpc(
"lock",
"bucket/object",
budget,
async move {
tokio::time::sleep(budget).await;
Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
},
RpcCleanup::new(None, late_failure, None),
)
.await;
assert!(
matches!(result, Err(LockError::Timeout { .. })),
"expired caller budget must retain timeout classification"
);
let health = lock_peer_health_for_test(addr);
assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
assert!(
health.last_eviction.is_none(),
"a recently serving peer keeps its channel after a caller timeout"
);
timeout(Duration::from_secs(1), released_rx)
.await
.expect("same-tick ambiguous completion must trigger cleanup")
.expect("cleanup signal");
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
0,
"same-tick completion returns detached admission"
);
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
let addr = "http://timeout-late-grant-during-eviction";
reset_lock_peer_health_for_test(addr);
let client = RemoteClient::new(addr.to_string());
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
eviction_pauses_for_test()
.lock()
.expect("eviction pause registry")
.insert(addr.to_string(), (evicting_tx, resume_rx));
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
Box::pin(async move {
released_tx.send(granted).expect("late grant observer remains alive");
})
}));
client
.execute_rpc(
"lock",
"bucket/object",
Duration::from_millis(25),
async move { Ok(grant_rx.await.expect("test controls the late grant")) },
RpcCleanup::new(late, None, None),
)
.await
});
timeout(Duration::from_secs(1), evicting_rx)
.await
.expect("timed-out RPC must reach channel eviction")
.expect("eviction signal");
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
1,
"eviction waits with a detached RPC owner"
);
grant_tx
.send(true)
.expect("timed-out stream remains available for a late grant");
assert!(
timeout(Duration::from_secs(1), released_rx)
.await
.expect("late grant must be cleaned up before eviction resumes")
.expect("late cleanup result")
);
assert!(!caller.is_finished(), "channel eviction is still paused");
assert_eq!(
lock_peer_health_for_test(addr).detached_rpcs,
0,
"late completion returns detached admission"
);
resume_tx.send(()).expect("resume channel eviction");
let result = caller.await.expect("timed-out caller completes without panicking");
assert!(
matches!(result, Err(LockError::Timeout { .. })),
"caller preserves its RPC timeout result"
);
reset_lock_peer_health_for_test(addr);
}
#[tokio::test]
async fn test_remote_client_success_polls_rpc_on_callers_task() {
tokio::spawn(async {
let client = RemoteClient::new("http://direct-poll-success".to_string());
let caller_id = tokio::task::id();
let rpc_id = client
.execute_rpc(
"ping",
"health-lock-online",
Duration::from_secs(1),
async { Ok(tokio::task::id()) },
RpcCleanup::new(None, None, None),
)
.await
.expect("successful RPC must return its result");
assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
})
.await
.expect("caller task must complete");
}
#[tokio::test]
async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let caller = tokio::spawn(async move {
let client = RemoteClient::new("http://direct-poll-panic".to_string());
let late_failure: LateFailure = Some(Box::new(move |()| {
Box::pin(async move {
released_tx.send(()).expect("panic cleanup observer remains alive");
})
}));
client
.execute_rpc::<(), _>(
"lock",
"bucket/object",
Duration::from_secs(1),
async { panic!("test peer RPC panic") },
RpcCleanup::new(None, late_failure, None),
)
.await
});
let result = caller.await.expect("RPC panic must not escape to its caller");
assert!(
matches!(result, Err(LockError::Internal { .. })),
"RPC panic remains an internal lock failure"
);
timeout(Duration::from_secs(1), released_rx)
.await
.expect("panic must trigger ambiguous-grant cleanup")
.expect("cleanup signal");
}
#[tokio::test]
#[serial_test::serial]
async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
use std::path::PathBuf;
use tokio::fs;
use uuid::Uuid;
struct PhysicalFixture {
_directories: Vec<tempfile::TempDir>,
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
);
}
}
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
let fixture = fixture(0).await;
for pool in disks(&fixture.store).await {
for disk in pool {
let object = object_path(&disk);
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
if entry.file_type().await.expect("fixture entry type").is_dir() {
fs::remove_dir_all(entry.path())
.await
.expect("remove task-owned fixture rollback directory");
}
}
}
}
fixture
}
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
let mut all = Vec::new();
for pool in &store.pools {
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
let members = set
.disks
.read()
.await
.iter()
.cloned()
.map(|disk| disk.expect("identity fixture disk must be online"))
.collect::<Vec<_>>();
let mut heads = Vec::new();
for disk in members {
match disk
.pool_meta_native_recovery(Request::InspectIdentity {})
.await
.expect("physical identity inspection")
{
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
_ => panic!("identity inspection must return its physical head"),
}
}
all.push(heads);
}
all
}
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
let fixture = owner_repair_no_backups_fixture().await;
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("fixture must provide valid committed pool metadata"),
};
meta.version = version;
let raw = meta
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
.expect("legacy fixture should encode in its original version");
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
let all = disks(&fixture.store).await;
let now = OffsetDateTime::now_utc();
for pool in &all {
let heads = replicas(&raw, now).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install exact legacy head");
}
}
if stale {
let old = replicas(&raw, now - Duration::seconds(10)).await;
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
.await
.expect("install stale same-raw legacy head");
}
(fixture, raw)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
drop(state);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for read in &reads {
assert!(
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
let mut source_raw = raw.clone();
// A second MessagePack value is readable by legacy decoding but
// must not be copied as part of an unverified repair payload.
source_raw.push(0xc0);
assert!(
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
"ordinary legacy reads must remain compatible with trailing bytes"
);
let all_disks = disks(&fixture.store).await;
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in all_disks[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install readable legacy source with trailing bytes");
}
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await
.expect("legacy semantic selection must remain readable");
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
drop(state);
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"version {version}: no identity shard may change"
);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
for target_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
_ => panic!("initial V1 metadata must decode"),
};
meta.version = POOL_META_VERSION;
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V1 payload must remain valid"),
};
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V2 payload must remain valid"),
};
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
// Pool two follows pool zero; pools zero and one cover both
// orderings and both target formats in separate runs.
let versions = [first_version, other_version, first_version];
let physical = disks(&fixture.store).await;
let timestamp = OffsetDateTime::now_utc();
for (pool_index, pool) in physical.iter().enumerate() {
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
let heads = replicas(raw, timestamp).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
}
}
let before = snapshots(&fixture.store).await;
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await.expect("equivalent legacy pools should be selectable");
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
tokio::time::timeout(std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
.await.expect("mixed legacy heal must terminate")
.expect("equivalent legacy metadata should heal without downgrading the format floor");
drop(state);
let after = snapshots(&fixture.store).await;
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for (pool_index, read) in reads.iter().enumerate() {
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
assert!(matches!(&read.replica,
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
for (index, snapshot) in after[pool_index].iter().enumerate() {
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
.await.expect("every physical shard must match its selected legacy payload");
}
if pool_index != target_pool {
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
}
}
}
}
},
).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for prepare_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid {
meta,
canonical,
revision,
committed: true,
..
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
_ => panic!("baseline V1 metadata must be committed and readable"),
};
let previous = PoolMetaCommittedCandidate {
canonical: legacy_canonical,
meta: legacy_meta.clone(),
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
};
let mut next = legacy_meta;
next.version = POOL_META_GENERATION_VERSION;
next.pools[0].last_update += Duration::seconds(1);
let revision = PoolMetaRevision {
version: POOL_META_GENERATION_VERSION,
cluster_id: fixture.revision.cluster_id,
epoch: fixture.revision.epoch,
generation: 1,
transaction_id: Some(Uuid::new_v4()),
};
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
.expect("V3 PREPARE with V1 predecessor must encode");
assert!(matches!(decode_pool_meta_replica(pending.clone()),
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
let physical = disks(&fixture.store).await;
for (index, disk) in physical[prepare_pool].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install task-owned V3 PREPARE evidence");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[prepare_pool].replica,
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
if raw == &pending && *actual == revision));
let other_pool = 1 - prepare_pool;
assert!(matches!(&reads[other_pool].replica,
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture
.store
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
)
.await
.expect("mixed V1/V3 PREPARE heal must terminate");
drop(state);
assert!(
result.is_err(),
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"pool {prepare_pool}: rejection must not mutate identity evidence"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("current reader must recognize the legacy source"),
};
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source && revision.version == POOL_META_V1_VERSION));
let physical = disks(&fixture.store).await;
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install legacy source on pool zero");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve every original head and backup"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
"the dangerous source bytes must not be copied into the target pool"
);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("safe pool-zero source must decode"),
};
assert!(
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
"the dangerous target must remain semantically equivalent to the safe source"
);
let physical = disks(&fixture.store).await;
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
.await
.expect("install legacy dangerous target head");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy target heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve all physical heads and backups"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
"unsafe target evidence must not be overwritten by the safe source"
);
},
)
.await;
}
+4 -128
View File
@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
#[cfg(test)]
use std::io::Cursor;
use std::io::Write;
use std::path::PathBuf;
@@ -6838,47 +6839,11 @@ impl Drop for PoolMetaTransactionArm {
}
}
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
if data.len() >= 4
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
// The selection already validates the payload. The runtime-only
// flag can only appear in the legacy three-field sequence or map.
&& rmp::decode::read_array_len(&mut &data[4..])
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
.ok()
== Some(3)
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
&& legacy.dont_save
{
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
}
Ok(())
}
// Ordinary legacy reads retain their compatibility behavior, but bytes that
// will be copied to another pool must not contain an unparsed extension.
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
let payload = data
.get(4..)
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
if decoder.get_ref().position() != payload_len {
return Err(Error::other("pool metadata legacy source has trailing bytes"));
}
Ok(())
}
#[derive(Debug)]
struct PoolMetaSelection {
meta: PoolMeta,
revision: PoolMetaRevision,
canonical: Option<Vec<u8>>,
// Legacy semantic comparison canonicalizes to V2; a heal must instead
// preserve verified raw bytes at the highest observed format floor.
legacy_raw: Option<Vec<u8>>,
legacy_has_unsafe_flag: bool,
replica_state: PoolMetaReplicaState,
cas_tokens: Vec<PoolMetaCasToken>,
absent: bool,
@@ -7269,8 +7234,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: PoolMeta::default(),
revision: PoolMetaRevision::legacy(0),
canonical: None,
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair: false,
repair_write_safe: true,
@@ -7325,8 +7288,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: selected.meta.clone(),
revision: selected.revision,
canonical: Some(selected.canonical.clone()),
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -7340,10 +7301,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
// record; divergent backups without it are ambiguous and fail closed.
let legacy_has_unsafe_flag = committed
.iter()
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
let mut floor_raw = None;
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
for (idx, raw, candidate) in committed {
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7357,36 +7314,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
}
} else {
needs_repair |= selected_raw != &raw;
if floor_raw.is_none()
&& candidate.revision.version == observed_version
&& raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(raw);
}
}
} else {
selected = Some((idx, raw, candidate));
}
}
let (_, selected_raw, mut selected) =
let (_, _, mut selected) =
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
if selected.revision.version == observed_version
&& selected_raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(selected_raw);
}
selected.meta.version = observed_version;
selected.revision.version = observed_version;
Ok(PoolMetaSelection {
meta: selected.meta,
revision: selected.revision,
canonical: Some(selected.canonical),
legacy_raw: floor_raw,
legacy_has_unsafe_flag,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -14606,23 +14546,6 @@ impl ECStore {
.canonical
.as_ref()
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
}
if selection.legacy_has_unsafe_flag {
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
}
let repair_data = if selection.revision.is_generation_protocol() {
canonical.clone()
} else {
let raw = selection
.legacy_raw
.clone()
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
ensure_legacy_pool_meta_payload_complete(&raw)?;
raw
};
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
let pool = self
.pools
.get(pool_index)
@@ -14633,10 +14556,10 @@ impl ECStore {
.get(pool_index)
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
let mut arm = state.arm_transaction();
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
.await?;
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
if confirmed != repair_data {
if confirmed != *canonical {
return Err(Error::other("pool metadata native heal readback diverged"));
}
fence.ensure_held()?;
@@ -29559,53 +29482,6 @@ mod tests {
);
}
#[test]
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
let frame = |payload: Vec<u8>| {
let mut data = Vec::new();
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
data.extend_from_slice(&payload);
data
};
let two_fields = PersistedPoolMetaV1 {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
};
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
let data = frame(payload.expect("two-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
for dont_save in [false, true] {
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
dont_save,
};
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
let data = frame(payload.expect("three-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
}
// This guard is only called for already-decoded replicas; malformed
// containers must fail selection rather than becoming repair sources.
let truncated = frame(vec![0x93, 0x01]);
assert!(matches!(
decode_pool_meta_replica(truncated),
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
));
}
#[test]
fn pool_meta_decode_supports_legacy_payload() {
let start_time = OffsetDateTime::now_utc();
+2 -3
View File
@@ -317,9 +317,8 @@ impl Sets {
&self.ctx
}
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
#[cfg(any(test, feature = "test-util"))]
pub async fn stop_endpoint_monitor_for_test(&self) {
#[cfg(test)]
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
if let Some(exit_signal) = &self.exit_signal {
let _ = exit_signal.send(());
// Wait for an in-flight reconnect pass as well as its receiver to exit.
+3 -98
View File
@@ -55,20 +55,6 @@ use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use uuid::Uuid;
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
/// Completion may mean legacy fallback after a failed scan; callers must assert
/// the expected index result separately. Remote disks have no local worker.
#[cfg(any(test, feature = "test-util"))]
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
match disk.as_ref() {
crate::disk::Disk::Local(local) => {
local.disk.wait_for_multipart_index_bootstrap().await;
Ok(())
}
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
}
}
/// Disk health status constants
const DISK_HEALTH_OK: u32 = 0;
const DISK_HEALTH_FAULTY: u32 = 1;
@@ -2457,7 +2443,6 @@ impl LocalDiskWrapper {
object: &str,
transaction: uuid::Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
self.track_disk_health_mutation(
"settle_delete_transaction",
@@ -2465,7 +2450,7 @@ impl LocalDiskWrapper {
|| {
Box::pin(async {
self.disk
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
})
},
@@ -2481,7 +2466,6 @@ impl LocalDiskWrapper {
transaction: uuid::Uuid,
journal: &crate::disk::DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
self.track_disk_health_mutation(
"settle_delete_transaction",
@@ -2489,14 +2473,7 @@ impl LocalDiskWrapper {
|| {
Box::pin(async {
self.disk
.try_finish_and_finalize_prepared_delete_local_with_budget(
volume,
object,
transaction,
journal,
namespace_owner,
work_budget,
)
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
})
},
@@ -2751,18 +2728,6 @@ impl LocalDiskWrapper {
#[async_trait::async_trait]
impl DiskAPI for LocalDiskWrapper {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.list_multipart_index_roots(bucket, marker).await
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
// The local implementation shares this binary's capability. Availability
// and disk identity remain mutation checks and write-quorum inputs.
@@ -2816,62 +2781,6 @@ impl DiskAPI for LocalDiskWrapper {
)
.await
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
self.track_disk_health_with_op(
"discover_delete_transactions",
|| async {
self.disk
.discover_delete_transactions_bounded(volume, object, max_entries)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: uuid::Uuid,
max_entries: u32,
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
self.track_disk_health_with_op(
"inspect_delete_transaction",
|| async {
self.disk
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: crate::disk::BoundedDeleteRecoverySettlement,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
self.track_disk_health_mutation(
"settle_delete_transaction",
DiskMetricMutation::Delete,
|| async {
self.disk
.settle_delete_transaction_bounded(volume, request, namespace_owner)
.await
},
crate::disk::get_delete_recovery_timeout()?,
)
.await
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -3840,11 +3749,7 @@ mod tests {
assert!(futures::poll!(&mut operation).is_pending());
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
assert!(futures::poll!(&mut operation).is_pending());
assert!(
!logs.contents().contains("disk_operation_slow"),
"normal disk latency should not warn: {}",
logs.contents()
);
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
time::advance(Duration::from_millis(1)).await;
assert!(futures::poll!(&mut operation).is_pending());
let warning = logs.contents();
-6
View File
@@ -172,8 +172,6 @@ pub enum DiskError {
#[error("guarded batch deletion is not supported by this disk")]
GuardedDeleteUnsupported,
#[error("foreground delete recovery work limit reached")]
DeleteRecoveryWorkLimit,
#[error("erasure write quorum")]
ErasureWriteQuorum,
@@ -772,7 +770,6 @@ impl Clone for DiskError {
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
DiskError::ShortWrite => DiskError::ShortWrite,
@@ -832,7 +829,6 @@ impl DiskError {
DiskError::RemoteClientUnavailable(_) => 0x2B,
DiskError::ConditionalDeleteConflict => 0x2C,
DiskError::GuardedDeleteUnsupported => 0x2D,
DiskError::DeleteRecoveryWorkLimit => 0x2E,
}
}
@@ -883,7 +879,6 @@ impl DiskError {
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
0x2C => Some(DiskError::ConditionalDeleteConflict),
0x2D => Some(DiskError::GuardedDeleteUnsupported),
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
_ => None,
}
}
@@ -1667,7 +1662,6 @@ mod tests {
for (error, code) in [
(DiskError::ConditionalDeleteConflict, 0x2C),
(DiskError::GuardedDeleteUnsupported, 0x2D),
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
] {
assert_eq!(error.to_u32(), code);
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
-456
View File
@@ -183,459 +183,3 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
assert_eq!(fresh.len, 11);
}
}
#[tokio::test]
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
let (_directory, file) = fixture();
let mut cache = FdCache::new();
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
cache.invalidate_under("v", "missing/prefix");
cache.invalidate_volume("v");
cache.invalidate_exact("v", "missing/prefix/part.1").await;
cache.clear();
assert!(
cache.maintenance_task.get().is_none(),
"empty-cache invalidations must not spawn maintenance"
);
// Populate a normal entry and verify matched vs unmatched filter gates directly,
// and ensure unmatched invalidations do not enqueue pending removals.
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
assert_eq!(cache.tracker.live_entries(), 1);
assert!(cache.tracker.may_match_volume("v"));
assert!(!cache.tracker.may_match_volume("other-volume"));
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
cache.invalidate_volume("other-volume");
cache.invalidate_under("v", "obj-2/0198a3f0");
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
assert!(
!cache
.tracker
.has_pending_invalidations
.load(std::sync::atomic::Ordering::SeqCst),
"unmatched invalidations must not enqueue maintenance removals"
);
assert!(cache.get(&cached_key).await.is_some());
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
// to per-volume unindexed_counts without degrading other volumes.
let other_vol_key = key("v2", "shallow/part.1", false);
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
let deep_key = key("v", deep_path, false);
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
assert_eq!(
cache
.tracker
.unindexed_counts
.iter()
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
.sum::<u32>(),
1
);
assert!(
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
"deep key in volume v must not disable prefix filtering in volume v2"
);
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
assert!(cache.get(&deep_key).await.is_none());
cache.invalidate_volume("v2");
assert!(cache.get(&other_vol_key).await.is_none());
// Replacing an invalidated key before maintenance drains pending_invalidations
// must release the old descriptor immediately and keep the replacement live
// even when a strong reference to the old CachedFdEntry shell survives until
// maintenance runs its Arc::ptr_eq check.
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
let old_weak = Arc::downgrade(&old_file);
cache.insert(cached_key.clone(), old_file).await;
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
cache.invalidate_under("v", "obj-1/0198a3f0");
assert!(cache.get(&cached_key).await.is_none());
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
assert!(
old_weak.upgrade().is_none(),
"replacing an entry must retire and drop the previous descriptor immediately"
);
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache
.tracker
.has_pending_invalidations
.load(std::sync::atomic::Ordering::SeqCst)
{
tokio::task::yield_now().await;
}
})
.await
.expect("maintenance should drain pending invalidations");
drop(held_old_shell);
assert!(
cache.get(&cached_key).await.is_some(),
"stale pending invalidation must not evict a newer replacement"
);
assert_eq!(
cache
.tracker
.unindexed_counts
.iter()
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
.sum::<u32>(),
0
);
// Dropping a cache while a live non-invalidated entry is present (even if a
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
// must retire the descriptor and release FdCacheTracker without cycles.
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
let live_drop_weak = Arc::downgrade(&live_drop_file);
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
let shutdown_weak = Arc::downgrade(&shutdown_file);
let tracker_weak = Arc::downgrade(&cache.tracker);
let cloned_moka = cache.cache.clone();
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
cache.invalidate_under("v", "shutdown");
if let Some(handle) = cache.maintenance_task.take() {
handle.abort();
let _ = handle.await;
}
drop(cache);
assert!(
live_drop_weak.upgrade().is_none(),
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
);
assert!(
shutdown_weak.upgrade().is_none(),
"dropping cache when maintenance is aborted must still release descriptors"
);
assert!(
tracker_weak.upgrade().is_none(),
"dropping cache must release FdCacheTracker without reference cycles"
);
drop(cloned_moka);
// Short-TTL expired entries (including mixed expired and fresh entries) must
// be retired by invalidate_under and clear() even though Moka's iterator skips
// expired entries.
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
let expired_weak = Arc::downgrade(&expired_file);
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
let fresh_weak = Arc::downgrade(&fresh_file);
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
ttl_cache.invalidate_under("v", "ttl-obj");
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("invalidate_under must retire both expired and fresh matching descriptors");
assert_eq!(ttl_cache.tracker.live_entries(), 0);
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
ttl_cache
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
.await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
let clear_live_weak = Arc::downgrade(&clear_live_file);
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
ttl_cache.clear();
assert!(
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
"clear() must synchronously retire both TTL-expired and live descriptors"
);
assert_eq!(ttl_cache.tracker.live_entries(), 0);
}
#[tokio::test]
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
use std::collections::HashMap;
use std::sync::atomic::Ordering;
let (_directory, file) = fixture();
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
let paths = [
"a/b",
"a/b/part.1",
"a/b/dir/part.2",
"a/bc",
"a/bc/part.1",
"prefix/obj-0001.bin/0198a3f0/part.1",
"prefix/obj-0002.bin/0198a3f0/part.1",
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
];
let prefixes = [
"a",
"a/",
"a/b",
"a/b/",
"a/b/dir",
"a/bc",
"prefix",
"prefix/obj-0001.bin",
"prefix/obj-0001.bin/0198a3f0",
"d1/d2/d3/d4/d5/c_missing",
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
"nonexistent/prefix",
];
let mut candidate_keys = Vec::new();
for v in volumes {
for p in paths {
for direct in [false, true] {
candidate_keys.push(key(v, p, direct));
}
}
}
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
let mut next_u64 = || {
rng ^= rng << 13;
rng ^= rng >> 7;
rng ^= rng << 17;
rng
};
for step in 0..1000 {
let op = next_u64() % 100;
if op < 45 {
// Insert or replace (either fresh or stale generation)
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
let entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: step,
invalidated: AtomicBool::new(false),
});
let cur_gen = cache.generation();
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
if !stale {
oracle.insert(k, entry);
}
} else if op < 65 {
// Exact invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
let p = paths[(next_u64() as usize) % paths.len()];
cache.invalidate_exact(v, p).await;
oracle.remove(&key(v, p, false));
oracle.remove(&key(v, p, true));
} else if op < 88 {
// Prefix invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
cache.invalidate_under(v, raw_prefix);
let trimmed = raw_prefix.trim_end_matches('/');
oracle.retain(|k, _| {
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
});
} else if op < 97 {
// Volume invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
cache.invalidate_volume(v);
oracle.retain(|k, _| k.volume != v);
} else {
// Full clear
cache.clear();
oracle.clear();
}
// Verify lockstep equivalence against the reference oracle after every operation.
for k in &candidate_keys {
let got = cache.get(k).await;
let expected = oracle.get(k);
match (got, expected) {
(Some(g), Some(e)) => assert!(
Arc::ptr_eq(&g, e),
"step {step}: pointer mismatch for key ({}, {}, {})",
k.volume,
k.path,
k.direct
),
(None, None) => {}
(g, e) => panic!(
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
k.volume,
k.path,
k.direct,
g.is_some(),
e.is_some()
),
}
}
}
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
cache.clear();
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("clear must drain all live entries and pending invalidations");
assert_eq!(cache.tracker.live_entries(), 0);
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
use std::sync::atomic::{AtomicU64, Ordering};
let (_directory, file) = fixture();
let cache = Arc::new(FdCache::new());
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
// at `entry_shards[shard].lock()` after it has passed the initial generation
// check and incremented `volume_counts`, bump generation via an unmatched
// invalidation, and verify the post-`new()` recheck refuses publication and
// preserves the existing entry.
let seam_key = key("v-seam", "obj-1/part.1", false);
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
let stale_gen = cache.generation();
let stale_entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: 999_999,
invalidated: AtomicBool::new(false),
});
let blocked_insert = {
let hold = cache.tracker.entry_shards[entry_shard].lock();
let cache_clone = Arc::clone(&cache);
let seam_key_clone = seam_key.clone();
let task = tokio::spawn(async move {
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
});
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
assert!(
std::time::Instant::now() < deadline,
"insert never reached CachedFdEntry::new with the previous entry still live"
);
std::thread::yield_now();
}
cache.invalidate_under("v-seam", "unmatched-prefix");
drop(hold);
task
};
blocked_insert.await.expect("blocked insert task should complete");
let preserved = cache
.get(&seam_key)
.await
.expect("refused stale insert must not retire a valid previous entry");
assert!(Arc::ptr_eq(&preserved, &initial_entry));
cache.invalidate_volume("v-seam");
let floor_v1 = Arc::new(AtomicU64::new(0));
let floor_v2 = Arc::new(AtomicU64::new(0));
let mut handles = Vec::new();
for t in 0..8usize {
let cache = Arc::clone(&cache);
let file = Arc::clone(&file);
let floor_v1 = Arc::clone(&floor_v1);
let floor_v2 = Arc::clone(&floor_v2);
handles.push(tokio::spawn(async move {
for i in 0..250u64 {
let is_v1 = (t + i as usize).is_multiple_of(2);
let vol = if is_v1 { "v1" } else { "v2" };
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
let k = key(vol, &path, i.is_multiple_of(2));
let g = cache.generation();
let entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: g,
invalidated: AtomicBool::new(false),
});
cache.insert_if_fresh(k.clone(), entry, g).await;
let min_gen = if is_v1 {
floor_v1.load(Ordering::SeqCst)
} else {
floor_v2.load(Ordering::SeqCst)
};
if let Some(hit) = cache.get(&k).await {
assert!(
hit.len >= min_gen,
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
hit.len
);
}
}
}));
}
for t in 0..4usize {
let cache = Arc::clone(&cache);
let floor_v1 = Arc::clone(&floor_v1);
let floor_v2 = Arc::clone(&floor_v2);
handles.push(tokio::spawn(async move {
for i in 0..250u64 {
let is_v1 = (t + i as usize).is_multiple_of(2);
let vol = if is_v1 { "v1" } else { "v2" };
match i % 4 {
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
1 => {
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
cache.invalidate_exact(vol, &p).await;
}
2 => {
let before = cache.generation();
cache.invalidate_volume(vol);
let floor = before + 1;
if is_v1 {
floor_v1.fetch_max(floor, Ordering::SeqCst);
} else {
floor_v2.fetch_max(floor, Ordering::SeqCst);
}
}
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
}
tokio::task::yield_now().await;
}
}));
}
for h in handles {
h.await.expect("worker task should succeed");
}
cache.clear();
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("clear after concurrent stress must drain all live entries");
assert_eq!(cache.tracker.live_entries(), 0);
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
}
File diff suppressed because it is too large Load Diff
-3
View File
@@ -486,9 +486,6 @@ impl LocalDisk {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
})?;
crate::hp_guard!("LocalDisk::rename_data");
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
self.multipart_index.record_info(dst_path, &fi).await?;
}
#[cfg(feature = "e2e-test-hooks")]
if fi.is_healing() {
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
+38 -557
View File
@@ -5,28 +5,6 @@ use crate::disk::{
};
use sha2::{Digest, Sha256};
pub(super) struct RecoveryOperationScope {
pub(super) budget: Arc<crate::object_api::DeleteRecoveryWorkBudget>,
may_publish_receipt: bool,
receipt_published: std::sync::atomic::AtomicBool,
}
tokio::task_local! {
static RECOVERY_OPERATION: Arc<RecoveryOperationScope>;
}
pub(super) fn recovery_operation_scope() -> Option<Arc<RecoveryOperationScope>> {
RECOVERY_OPERATION.try_with(Arc::clone).ok()
}
impl RecoveryOperationScope {
pub(super) fn receipt_published(&self) {
if self.may_publish_receipt {
self.receipt_published.store(true, std::sync::atomic::Ordering::Release);
}
}
}
pub(super) const PREPARED: &str = "delete-prepared";
pub(super) const TARGET: &str = "delete-target";
pub(super) const FINISHED: &str = "delete-commit-finished";
@@ -41,35 +19,24 @@ struct DiscoverySession {
touched: Instant,
}
/// Plain (non-transaction) children of an object-scoped discovery root cost one
/// budget entry per this many names: enough to bound a hostile prefix without
/// making a real folder prefix undeletable.
const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
impl DiscoverySession {
/// Consume one bounded step, returning ownership only when traversal remains.
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
.map(|(transactions, next, _)| (transactions, next))
}
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
fn advance(mut self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
if self.frames.is_empty() {
match std::fs::read_dir(&self.root) {
Ok(entries) => self.frames.push((String::new(), entries)),
Err(err) if err.kind() == ErrorKind::NotFound => {
return Ok((Vec::new(), None, 0));
return Ok((Vec::new(), None));
}
Err(err) => return Err(err),
}
}
let mut transactions = BTreeSet::new();
let mut seen_candidates = HashSet::new();
let mut visited = 0u32;
let mut plain_children = 0u32;
let mut visited = 0usize;
while !self.frames.is_empty()
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
&& visited < max_entries
&& visited < 4096
{
let (parent, entries) = self
.frames
@@ -82,6 +49,7 @@ impl DiscoverySession {
continue;
}
};
visited += 1;
let name = entry
.file_name()
.into_string()
@@ -89,19 +57,6 @@ impl DiscoverySession {
if !safe_path(&name, false) || name.contains('/') {
return Err(std::io::Error::from(ErrorKind::InvalidData));
}
// An object-scoped scan only ever descends into transaction (UUID)
// directories at the root. Plain child keys of a prefix directory are
// skipped by name without a stat; they are charged at one entry per
// readdir batch so the work stays bounded while deleting a missing key
// that names a prefix with thousands of children can still complete.
if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
plain_children += 1;
if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
visited += 1;
}
continue;
}
visited += 1;
let relative = if parent.is_empty() {
name.clone()
} else {
@@ -201,7 +156,7 @@ impl DiscoverySession {
.map(|(object, transaction)| DeleteTransactionId { object, transaction })
.collect();
let next = (!self.frames.is_empty()).then_some(self);
Ok((transactions, next, visited))
Ok((transactions, next))
}
async fn has_evidence(mut self, disk: &LocalDisk, path: &Path) -> Result<bool> {
@@ -625,41 +580,12 @@ fn explicit_delete_referenced_data_dirs(
Ok(Some(referenced))
}
pub(super) async fn next_recovery_entry(entries: &mut tokio::fs::ReadDir) -> std::io::Result<Option<tokio::fs::DirEntry>> {
if let Some(scope) = recovery_operation_scope() {
scope.budget.reserve(1).map_err(std::io::Error::from)?;
}
entries.next_entry().await
}
pub(super) fn recovery_entries(
mut entries: std::fs::ReadDir,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> impl Iterator<Item = std::io::Result<std::fs::DirEntry>> + '_ {
let mut finished = false;
std::iter::from_fn(move || {
if finished {
return None;
}
if let Some(budget) = budget
&& let Err(error) = budget.reserve(1)
{
finished = true;
return Some(Err(error.into()));
}
let entry = entries.next();
finished = entry.is_none();
entry
})
}
fn explicit_delete_plan_bounded(
fn explicit_delete_plan(
root: &Path,
transaction: Uuid,
original: Option<&FileMeta>,
journal: Option<&DeleteRecoveryJournal>,
current_missing: bool,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> Result<Option<ExplicitDeletePlan>> {
let Some(referenced) = explicit_delete_referenced_data_dirs(original, journal, current_missing)? else {
return Ok(None);
@@ -676,7 +602,7 @@ fn explicit_delete_plan_bounded(
let mut candidates = Vec::new();
match std::fs::read_dir(&rollback) {
Ok(entries) => {
for entry in recovery_entries(entries, budget) {
for entry in entries {
let entry = entry.map_err(to_file_error)?;
let kind = entry.file_type().map_err(to_file_error)?;
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
@@ -708,7 +634,7 @@ fn explicit_delete_plan_bounded(
}
match std::fs::read_dir(root) {
Ok(entries) => {
for entry in recovery_entries(entries, budget) {
for entry in entries {
let entry = entry.map_err(to_file_error)?;
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil()) else {
@@ -732,7 +658,7 @@ fn explicit_delete_plan_bounded(
candidates.dedup();
for dir in candidates {
let mut has_parts = false;
for entry in recovery_entries(std::fs::read_dir(&dir).map_err(to_file_error)?, budget) {
for entry in std::fs::read_dir(&dir).map_err(to_file_error)? {
let entry = entry.map_err(to_file_error)?;
if !entry.file_type().map_err(to_file_error)?.is_file() {
return Ok(None);
@@ -771,27 +697,15 @@ fn canonical_uuid(name: &str) -> Option<Uuid> {
Uuid::parse_str(name).ok().filter(|id| name == id.hyphenated().to_string())
}
pub(super) fn missing_marker_preserves_child_prefix(journal: &DeleteRecoveryJournal) -> bool {
journal.original_digest.is_none()
&& journal.original_versions.is_empty()
&& journal.data_dirs.is_empty()
&& journal.target_digest.is_some()
&& !journal.requests.is_empty()
&& journal
.requests
.iter()
.all(|request| request.deleted && request.version_id.is_some_and(|id| !id.is_nil()))
}
pub(super) fn inspection_with_markers_bounded(
pub(super) fn inspection_with_markers(
root: &Path,
volume: &str,
object: &str,
transaction: Uuid,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> Result<(DeleteRecoveryInspection, bool)> {
let rollback = root.join(transaction.to_string());
let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?;
let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?;
let prepared = regular_bytes(&rollback.join(PREPARED))?;
let target = regular_metadata_bytes(&rollback.join(TARGET))?;
let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?;
@@ -825,20 +739,6 @@ pub(super) fn inspection_with_markers_bounded(
{
return Err(DiskError::FileCorrupt);
}
// A failed creation of a marker over a child named `xl.meta` did not
// replace that directory. Older releases nevertheless left a journal.
// Only this bound, originally absent generation may treat it as absence.
let current_path = root.join(STORAGE_FORMAT_FILE);
let current = if !committed
&& before.is_none()
&& (journal.as_ref().is_some_and(missing_marker_preserves_child_prefix)
|| (journal.is_none() && target.is_none() && receipt_bytes.is_none()))
&& std::fs::symlink_metadata(&current_path).is_ok_and(|metadata| metadata.is_dir())
{
None
} else {
regular_metadata_bytes(&current_path)?
};
let backup_digest = before.as_deref().map(digest);
let current_digest = current.as_deref().map(digest);
let prepared_digest = prepared.as_deref().map(digest);
@@ -881,7 +781,7 @@ pub(super) fn inspection_with_markers_bounded(
Err(err) => return Err(to_file_error(err).into()),
};
if let Some(entries) = entries {
for entry in recovery_entries(entries, budget) {
for entry in entries {
let entry = entry.map_err(to_file_error)?;
let kind = entry.file_type().map_err(to_file_error)?;
if !kind.is_file() && !kind.is_dir() {
@@ -898,7 +798,7 @@ pub(super) fn inspection_with_markers_bounded(
descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len()));
if kind.is_dir() && id.is_some_and(|id| !id.is_nil()) {
has_other_uuid_dir |= id != Some(transaction);
for child in recovery_entries(std::fs::read_dir(entry.path()).map_err(to_file_error)?, budget) {
for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? {
let child = child.map_err(to_file_error)?;
let kind = child.file_type().map_err(to_file_error)?;
if !kind.is_dir() && !kind.is_file() {
@@ -944,7 +844,7 @@ pub(super) fn inspection_with_markers_bounded(
}
if name == transaction.to_string() && kind.is_dir() {
has_staged_dir = true;
for file in recovery_entries(std::fs::read_dir(child.path()).map_err(to_file_error)?, budget) {
for file in std::fs::read_dir(child.path()).map_err(to_file_error)? {
let file = file.map_err(to_file_error)?;
let kind = file.file_type().map_err(to_file_error)?;
let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
@@ -1068,8 +968,7 @@ pub(super) fn inspection_with_markers_bounded(
.is_some_and(|referenced| !referenced.contains(&transaction))
}
DeleteRecoveryState::LegacyAmbiguous | DeleteRecoveryState::NewPrepared => {
explicit_delete_plan_bounded(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none(), budget)?
.is_some()
explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some()
}
_ => false,
};
@@ -1103,17 +1002,21 @@ pub(super) fn inspection_with_markers_bounded(
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
let mut map = PREPARED_DELETE_RECEIPTS.lock();
if map.len() >= 4096
&& let Some(evict_key) = map.keys().next().cloned()
{
map.remove(&evict_key);
}
map.insert(rollback, entry);
}
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
PREPARED_DELETE_RECEIPTS.remove(rollback)
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
}
impl LocalDisk {
@@ -1140,7 +1043,6 @@ impl LocalDisk {
let volume = volume.to_owned();
let object = object.to_owned();
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
let recovery_scope = recovery_operation_scope();
os::run_blocking_namespace_operation(lease, move || {
Ok((|| {
let entries = match std::fs::read_dir(&root) {
@@ -1150,7 +1052,7 @@ impl LocalDisk {
};
let mut predecessors = BTreeSet::new();
let mut visited = 0usize;
for entry in recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())) {
for entry in entries {
visited += 1;
if visited > 65536 {
return Err(DiskError::DiskOngoingReq);
@@ -1303,42 +1205,6 @@ impl LocalDisk {
.map_err(|error| to_file_error(error).into())
}
#[cfg(test)]
pub(crate) fn public_delete_discovery_sessions_for_test(&self) -> usize {
DISCOVERY_SESSIONS
.lock()
.values()
.filter(|session| session.disk_root == self.io_root())
.count()
}
pub(super) async fn delete_recovery_discover_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<super::super::BoundedDeleteRecoveryDiscovery> {
if !(1..=super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES).contains(&max_entries) {
return Err(DiskError::InvalidPath);
}
let path = self.delete_discovery_path(volume, Some(object))?;
let session = self.new_delete_discovery_session(volume, Some(object), &path)?;
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
self.validate_delete_discovery_owner(&session)?;
// The blocking owner drops every directory handle even if the RPC waiter disappears.
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
os::run_blocking_namespace_operation(lease, move || {
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
Ok(super::super::BoundedDeleteRecoveryDiscovery {
transactions: if next.is_none() { transactions } else { Vec::new() },
visited,
complete: next.is_none(),
})
})
.await
.map_err(|error| to_file_error(error).into())
}
pub(super) async fn delete_recovery_discover(
&self,
volume: &str,
@@ -1386,39 +1252,6 @@ impl LocalDisk {
.await
}
pub(super) async fn delete_recovery_inspect_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<super::super::BoundedDeleteRecoveryInspection> {
if !safe_path(object, false)
|| transaction.is_nil()
|| !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&max_entries)
{
return Err(DiskError::InvalidPath);
}
let root = self.get_object_path(volume, object)?;
let owner = os::acquire_metadata_mutation_lease(&root, None).await;
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
let volume = volume.to_owned();
let object = object.to_owned();
os::run_blocking_namespace_operation(lease, move || {
let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
Ok(
inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
super::super::BoundedDeleteRecoveryInspection {
inspection,
visited: max_entries - budget.remaining(),
}
}),
)
})
.await
.map_err(to_file_error)?
}
pub(super) async fn delete_recovery_inspect(
&self,
volume: &str,
@@ -1458,22 +1291,9 @@ impl LocalDisk {
let volume = volume.to_owned();
let object = object.to_owned();
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
let scope = recovery_operation_scope();
// Journal inspections ride the durable-mutation lane (the fused delete
// prepare already does): a bulk delete publishes one transaction per
// key, and the one-worker foreground namespace lane would serialize
// those per-drive publishes behind every other mutation.
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
Ok(inspection_with_markers_bounded(
&root,
&volume,
&object,
transaction,
scope.as_ref().map(|scope| scope.budget.as_ref()),
))
})
.await
.map_err(to_file_error)?
os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
.await
.map_err(to_file_error)?
}
async fn activate_delete_preparation(
@@ -1671,49 +1491,6 @@ impl LocalDisk {
self.activate_delete_preparation(&journal, original, target, owner).await
}
pub(super) async fn delete_recovery_settle_bounded(
&self,
volume: &str,
request: super::super::BoundedDeleteRecoverySettlement,
owner: Option<Arc<dyn Send + Sync>>,
) -> Result<super::super::BoundedDeleteRecoverySettlementReply> {
if !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&request.max_entries) {
return Err(DiskError::InvalidPath);
}
let scope = Arc::new(RecoveryOperationScope {
budget: Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(request.max_entries)),
may_publish_receipt: matches!(
&request.action,
DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. }
),
receipt_published: std::sync::atomic::AtomicBool::new(false),
});
let result = RECOVERY_OPERATION
.scope(
Arc::clone(&scope),
self.delete_recovery_settle(
volume,
&request.object,
request.transaction,
request.expected_fingerprint,
request.action,
owner,
),
)
.await;
let outcome = match result {
Ok(inspection) => super::super::DeleteRecoverySettlementOutcome::Inspected(Box::new(inspection)),
Err(DiskError::DeleteRecoveryWorkLimit) if scope.receipt_published.load(std::sync::atomic::Ordering::Acquire) => {
super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
}
Err(error) => return Err(error),
};
Ok(super::super::BoundedDeleteRecoverySettlementReply {
outcome,
visited: request.max_entries - scope.budget.remaining(),
})
}
pub(crate) async fn delete_recovery_settle(
&self,
volume: &str,
@@ -1912,26 +1689,14 @@ impl LocalDisk {
let plan_root = root.clone();
let missing = observed.current_digest.is_none();
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
let scope = recovery_operation_scope();
// Same durable-mutation lane as the journal inspection above: this
// plan closure runs inside every settle on the bulk-delete publish
// path.
let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
let plan = os::run_blocking_namespace_operation(lease, move || {
Ok((|| {
let before =
regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?
.as_deref()
.map(FileMeta::load)
.transpose()?;
explicit_delete_plan_bounded(
&plan_root,
transaction,
before.as_ref(),
None,
missing,
scope.as_ref().map(|scope| scope.budget.as_ref()),
)?
.ok_or(DiskError::FileCorrupt)
explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt)
})())
})
.await
@@ -2196,7 +1961,6 @@ impl LocalDisk {
let (rollback_buf, root_buf, volume_str, object_str) =
(rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
let publication_root = self.publication_root.clone();
let scope = recovery_operation_scope();
return os::run_blocking_namespace_operation_with_sync(receipt_lease, sync, move || {
let _publication_root = publication_root;
for (path, _) in &file_leases {
@@ -2229,14 +1993,8 @@ impl LocalDisk {
}
}
}
let inspected = inspection_with_markers_bounded(
&root_buf,
&volume_str,
&object_str,
transaction,
scope.as_ref().map(|scope| scope.budget.as_ref()),
)
.map(|(state, _)| state);
let inspected =
inspection_with_markers(&root_buf, &volume_str, &object_str, transaction).map(|(state, _)| state);
drop((file_leases, prune_dirs));
Ok(inspected)
})
@@ -2393,7 +2151,6 @@ impl LocalDisk {
let volume_str = volume.to_owned();
let object_str = object.to_owned();
let recovery_scope = recovery_operation_scope();
let res = os::run_blocking_namespace_operation_with_sync(
lease,
sync,
@@ -2440,9 +2197,7 @@ impl LocalDisk {
let mut predecessors = BTreeSet::new();
let mut sole_transaction = true;
if let Ok(entries) = std::fs::read_dir(&file_path) {
for (visited, entry) in
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
{
for (visited, entry) in entries.enumerate() {
if visited >= 65536 {
return Err(DiskError::DiskOngoingReq);
}
@@ -2655,25 +2410,12 @@ impl LocalDisk {
Ok(Some(()))
}
#[cfg(test)]
pub(crate) async fn try_publish_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
self.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, None)
.await
}
pub(crate) async fn try_publish_prepared_delete_local_with_budget(
&self,
volume: &str,
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) || transaction.is_nil() {
@@ -2752,7 +2494,7 @@ impl LocalDisk {
if sole_transaction {
match std::fs::read_dir(&root) {
Ok(entries) => {
for entry in recovery_entries(entries, work_budget.as_deref()) {
for entry in entries {
let entry = entry?;
let Ok(name) = entry.file_name().into_string() else {
sole_transaction = false;
@@ -2800,7 +2542,6 @@ impl LocalDisk {
.map_err(to_file_error)?
}
#[cfg(test)]
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
&self,
volume: &str,
@@ -2808,26 +2549,6 @@ impl LocalDisk {
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<bool> {
self.try_finish_and_finalize_prepared_delete_local_with_budget(
volume,
object,
transaction,
journal,
namespace_owner,
None,
)
.await
}
pub(crate) async fn try_finish_and_finalize_prepared_delete_local_with_budget(
&self,
volume: &str,
object: &str,
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
match res {
@@ -2894,7 +2615,7 @@ impl LocalDisk {
had_physical_data_dirs = true;
let mut removable = Vec::new();
let mut preserve_dir = false;
for entry in recovery_entries(entries, work_budget.as_deref()) {
for entry in entries {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
@@ -4398,100 +4119,6 @@ mod tests {
);
}
#[tokio::test]
async fn delete_marker_collision_rejects_new_journals_and_recovers_legacy_proof() {
let (temp, disk, _info, original) = fixture().await;
let root = disk.io_get_object_path("recovery", "object").expect("object path");
let staged_child = temp.path().join("child-fixture");
fs::rename(&root, &staged_child).await.expect("move intact object");
fs::create_dir(&root).await.expect("create parent prefix");
fs::rename(staged_child, root.join(STORAGE_FORMAT_FILE))
.await
.expect("seed an independent object named object/xl.meta");
let transaction = Uuid::new_v4();
let marker = FileInfo {
volume: "recovery".to_owned(),
name: "object".to_owned(),
version_id: Some(Uuid::new_v4()),
deleted: true,
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
disk.delete_version_recoverable(
"recovery",
"object",
marker.clone(),
true,
DeleteOptions {
old_data_dir: Some(transaction),
..Default::default()
},
)
.await
.expect_err("a child xl.meta directory cannot be replaced by a marker");
assert!(!root.join(transaction.to_string()).exists(), "rejection must precede journal publication");
// Reproduce the durable evidence written by releases that prepared a
// missing-object marker before attempting to replace the directory.
disk.prepare_delete_recovery(
"recovery",
"object",
None,
&[marker],
None,
DeletePreparationContext {
transaction,
bucket_incarnation: None,
namespace_owner: None,
},
)
.await
.expect("seed legacy prepared marker");
let journal_path = root.join(transaction.to_string()).join(PREPARED);
let journal_bytes = fs::read(&journal_path).await.expect("prepared journal");
let mut incompatible: DeleteRecoveryJournal = serde_json::from_slice(&journal_bytes).expect("journal");
incompatible.original_digest = Some([7; 32]);
fs::write(&journal_path, serde_json::to_vec(&incompatible).expect("serialize journal"))
.await
.expect("write incompatible evidence");
assert!(
disk.inspect_delete_transaction("recovery", "object", transaction)
.await
.is_err(),
"a journal that claims existing metadata cannot ignore a directory collision"
);
fs::write(&journal_path, journal_bytes)
.await
.expect("restore exact legacy proof");
let observed = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("inspect legacy proof");
assert_eq!(observed.state, DeleteRecoveryState::NewPrepared);
let recovered = disk
.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback)
.await
.expect("rollback legacy marker without unlinking the child directory");
assert_eq!(recovered.state, DeleteRecoveryState::Absent);
assert!(!root.join(transaction.to_string()).exists());
assert_eq!(
disk.read_all("recovery", "object/xl.meta/xl.meta")
.await
.expect("child metadata")
.as_ref(),
original
);
assert!(
disk.read_version("", "recovery", "object/xl.meta", "", &Default::default())
.await
.is_ok()
);
assert!(
discover_all_keys(&disk, Some("object")).await.is_empty(),
"no proof may keep replacement heal pending"
);
}
// Data movement copies a delete marker onto a target that holds no
// metadata yet. Rolling back that sub-quorum write removes the marker it
// published, and must not report the backup it never wrote as missing:
@@ -6135,152 +5762,6 @@ mod tests {
);
}
#[tokio::test]
async fn delete_recovery_bounded_inspection_charges_root_and_uuid_children() {
let (_temp, disk, _, _) = fixture().await;
let transaction = Uuid::new_v4();
apply(&disk, transaction).await;
let full = disk
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
.await
.expect("full bounded inspection");
assert!(full.visited > 1, "both root and transaction directory entries are charged");
let error = disk
.inspect_delete_transaction_bounded("recovery", "object", transaction, full.visited - 1)
.await
.expect_err("nested entry exceeds allowance");
assert_eq!(error, DiskError::DeleteRecoveryWorkLimit);
let unchanged = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("unchanged proof");
assert_eq!(full.inspection.fingerprint, unchanged.fingerprint);
}
#[tokio::test]
async fn delete_recovery_receipt_publication_distinguishes_deferred_inspection_from_write_failure() {
for write_fails in [false, true] {
let (_temp, disk, _, _) = fixture().await;
let transaction = Uuid::new_v4();
apply(&disk, transaction).await;
let full = disk
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
.await
.expect("measure preparation work");
let receipt = disk
.io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}"))
.expect("receipt path");
if write_fails {
let destination = receipt.clone();
set_owned_file_before_open(&receipt, move || std::fs::create_dir(&destination).expect("reject receipt rename"));
}
let result = disk
.settle_delete_transaction_bounded(
"recovery",
super::super::super::BoundedDeleteRecoverySettlement {
object: "object".to_owned(),
transaction,
expected_fingerprint: full.inspection.fingerprint,
action: DeleteRecoveryAction::PublishCommit,
max_entries: full.visited,
},
None,
)
.await;
if write_fails {
assert!(result.is_err(), "a failed receipt write never counts as publication");
assert!(receipt.is_dir());
} else {
let result = result.expect("persisted receipt is acknowledged independently of post-write inspection");
assert!(matches!(
result.outcome,
super::super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
));
assert_eq!(result.visited, full.visited);
let state = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("durable receipt");
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
}
}
}
#[tokio::test]
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
let (_temp, disk, _, _) = fixture().await;
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
fs::create_dir_all(&path).await.expect("physical prefix");
// Transaction-shaped children are the entries a bounded probe pays for.
for _ in 0..64 {
fs::create_dir(path.join(Uuid::new_v4().to_string()))
.await
.expect("legitimate child");
}
for _ in 0..140 {
let page = disk
.discover_delete_transactions_bounded("recovery", "prefix", 8)
.await
.expect("bounded probe");
assert_eq!(page.visited, 8);
assert!(!page.complete, "an empty partial page cannot establish absence");
assert!(page.transactions.is_empty());
}
let page = disk
.discover_delete_transactions_bounded("recovery", "prefix", 128)
.await
.expect("complete probe");
assert!(page.complete);
assert_eq!(page.visited, 64);
assert!(page.transactions.is_empty());
let transaction = Uuid::new_v4();
apply(&disk, transaction).await;
let before = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("before probe");
let page = disk
.discover_delete_transactions_bounded("recovery", "object", 4096)
.await
.expect("complete evidence");
assert!(page.complete);
assert!(page.transactions.iter().any(|id| id.transaction == transaction));
let after = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("after probe");
assert_eq!(before.fingerprint, after.fingerprint);
assert!(
DISCOVERY_SESSIONS
.lock()
.values()
.all(|session| session.disk_root != disk.io_root())
);
}
#[tokio::test]
async fn delete_recovery_cancelled_bounded_discovery_drops_private_session() {
let (_temp, disk, _, _) = fixture().await;
let path = disk.delete_discovery_path("recovery", Some("object")).expect("probe scope");
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
let mut probe = Box::pin(disk.discover_delete_transactions_bounded("recovery", "object", 4096));
assert!(futures::poll!(probe.as_mut()).is_pending());
drop(probe);
assert!(
DISCOVERY_SESSIONS
.lock()
.values()
.all(|session| session.disk_root != disk.io_root())
);
drop(lease);
assert!(
disk.discover_delete_transactions_bounded("recovery", "object", 4096)
.await
.expect("subsequent probe")
.complete
);
}
#[tokio::test]
async fn delete_recovery_evidence_probes_do_not_register_public_cursors() {
let (_temp, disk, _, _) = fixture().await;
File diff suppressed because it is too large Load Diff
@@ -117,7 +117,6 @@ pub(super) struct LoadedSource {
}
pub(super) struct SourceStore {
multipart_index: Arc<super::multipart_index::IndexOwner>,
pub disk_id: Uuid,
pub physical_slot: usize,
pub disk_ids: Vec<Uuid>,
@@ -187,7 +186,6 @@ impl LocalDisk {
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
Ok(SourceStore {
multipart_index: self.multipart_index.clone(),
disk_id,
physical_slot,
disk_ids,
@@ -272,17 +270,21 @@ struct ActiveMultipartSourceEntry {
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
}
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
let source = source.as_ref();
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
return;
};
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
if map.len() >= 4096
&& let Some(evict_key) = map.keys().next().cloned()
{
map.remove(&evict_key);
}
map.insert(
source,
ActiveMultipartSourceEntry {
raw,
@@ -302,50 +304,56 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
}
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
let (raw, fi, head_stamp) =
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
let (raw, fi, head_stamp) = {
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let entry = map.get(source)?;
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return None;
}
Some((raw, fi))
}
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
return false;
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return false;
}
true
}
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
let Some((head_stamp, clean)) =
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
.lock()
.get(source)
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
else {
return false;
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return false;
}
clean
}
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
entry.unsettled_parts.insert(part_number);
entry.parts.remove(&part_number);
});
}
}
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
.with_get(source, |entry| entry.fi.data_dir)
.flatten()
.lock()
.get(source)
.and_then(|entry| entry.fi.data_dir)
.filter(|dir| !dir.is_nil())
else {
return;
@@ -354,9 +362,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
return;
};
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
entry.parts.insert(part.number, (part_stamp, part));
});
}
}
pub(super) fn finish_active_multipart_part_transaction(
@@ -365,29 +373,34 @@ pub(super) fn finish_active_multipart_part_transaction(
committed: bool,
retired_existing_txn: bool,
) {
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let Some(part_number) = part_number.filter(|_| committed) else {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
map.remove(source);
return;
};
if !retired_existing_txn {
return;
}
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
if let Some(entry) = map.get_mut(source)
&& !entry.unsettled_parts.remove(&part_number)
{
map.remove(source);
}
}
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
.with_get(source, |entry| {
if !entry.unsettled_parts.is_empty() {
return None;
}
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
let (expected_stamp, part) = entry.parts.get(&part_number)?;
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
})
.flatten()?;
let (head_stamp, data_dir, expected_stamp, part) = {
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let entry = map.get(source)?;
if !entry.unsettled_parts.is_empty() {
return None;
}
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
let (expected_stamp, part) = entry.parts.get(&part_number)?;
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return None;
}
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
@@ -398,11 +411,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
}
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
}
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
}
/// Canonical source mutations are admitted under SourceRW at the syscall,
@@ -757,9 +770,6 @@ impl SourceStore {
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
}
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
self.multipart_index
.record_head(&format!("{}/xl.meta", self.path), bytes)
.await?;
invalidate_active_multipart_source_head(&self.source);
let path = self.source.join(STORAGE_FORMAT_FILE);
let base = self.base.clone();
@@ -1926,7 +1936,6 @@ impl NativeMultipartPublication {
*self.validated_prepared.lock() = Some(validated);
Ok(())
}
#[cfg(target_os = "linux")]
pub(super) fn prepare_and_install_head_on_thread(
&self,
new_metadata: &[u8],
+4 -170
View File
@@ -27,7 +27,6 @@ mod io_admission;
#[cfg(test)]
pub(crate) mod lifecycle_test_io;
pub mod local;
pub mod multipart_index;
pub mod multipart_source;
pub mod os;
@@ -415,11 +414,10 @@ impl Disk {
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
match self {
Self::Local(disk) => {
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
}
Self::Remote(_) => Ok(None),
@@ -433,19 +431,11 @@ impl Disk {
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
match self {
Self::Local(disk) => {
disk.try_finish_and_finalize_prepared_delete_local(
volume,
object,
transaction,
journal,
namespace_owner,
work_budget,
)
.await
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
}
Self::Remote(_) => Ok(false),
}
@@ -522,17 +512,6 @@ impl Disk {
#[async_trait::async_trait]
impl DiskAPI for Disk {
async fn list_multipart_index_roots(
&self,
bucket: &str,
marker: Option<&str>,
) -> Result<multipart_index::MultipartIndexPage> {
match self {
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
}
}
async fn supports_guarded_delete_versions(&self) -> Result<()> {
match self {
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
@@ -576,17 +555,6 @@ impl DiskAPI for Disk {
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
}
}
async fn discover_delete_transactions_bounded(
&self,
volume: &str,
object: &str,
max_entries: u32,
) -> Result<BoundedDeleteRecoveryDiscovery> {
match self {
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
}
}
async fn discover_delete_transactions(
&self,
volume: &str,
@@ -599,35 +567,6 @@ impl DiskAPI for Disk {
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
}
}
async fn settle_delete_transaction_bounded(
&self,
volume: &str,
request: BoundedDeleteRecoverySettlement,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<BoundedDeleteRecoverySettlementReply> {
match self {
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
}
}
async fn inspect_delete_transaction_bounded(
&self,
volume: &str,
object: &str,
transaction: Uuid,
max_entries: u32,
) -> Result<BoundedDeleteRecoveryInspection> {
match self {
Self::Local(disk) => {
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
}
Self::Remote(disk) => {
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
.await
}
}
}
async fn inspect_delete_transaction(
&self,
volume: &str,
@@ -1574,15 +1513,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
fn host_name(&self) -> String;
fn endpoint(&self) -> Endpoint;
async fn close(&self) -> Result<()>;
/// Return candidates only while this disk maintains complete boot-local coverage.
/// Unsupported or unready disks must never return an authoritative empty page.
async fn list_multipart_index_roots(
&self,
_bucket: &str,
_marker: Option<&str>,
) -> Result<multipart_index::MultipartIndexPage> {
Err(DiskError::MethodNotAllowed)
}
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
@@ -1760,33 +1690,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
) -> Result<DeleteRecoveryPage> {
Err(DiskError::MethodNotAllowed)
}
/// Bounded foreground inventory owns all traversal state until return or cancellation.
/// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
async fn discover_delete_transactions_bounded(
&self,
_volume: &str,
_object: &str,
_max_entries: u32,
) -> Result<BoundedDeleteRecoveryDiscovery> {
Err(DiskError::DeleteRecoveryWorkLimit)
}
async fn inspect_delete_transaction_bounded(
&self,
_volume: &str,
_object: &str,
_transaction: Uuid,
_max_entries: u32,
) -> Result<BoundedDeleteRecoveryInspection> {
Err(DiskError::DeleteRecoveryWorkLimit)
}
async fn settle_delete_transaction_bounded(
&self,
_volume: &str,
_request: BoundedDeleteRecoverySettlement,
_namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<BoundedDeleteRecoverySettlementReply> {
Err(DiskError::DeleteRecoveryWorkLimit)
}
/// A read-only capability check; unsupported implementations must fail closed.
async fn supports_guarded_delete_versions(&self) -> Result<()> {
Err(DiskError::GuardedDeleteUnsupported)
@@ -2172,13 +2075,6 @@ pub struct WalkDirOptions {
#[serde(default)]
pub skip_hidden_prefix_check: bool,
/// Bound hidden metadata read while proving a non-recursive prefix visible.
#[serde(default)]
pub hidden_entry_budget: usize,
/// A prefix whose visibility was not established before the preceding cut.
#[serde(default)]
pub pending_prefix: Option<String>,
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
pub report_notfound: bool,
@@ -2511,62 +2407,9 @@ pub enum DeleteRecoveryAction {
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
}
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoveryDiscovery {
pub transactions: Vec<DeleteTransactionId>,
pub visited: u32,
pub complete: bool,
}
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoverySettlement {
pub object: String,
pub transaction: Uuid,
pub expected_fingerprint: DeleteRecoveryDigest,
pub action: DeleteRecoveryAction,
pub max_entries: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub enum DeleteRecoverySettlementOutcome {
Inspected(Box<DeleteRecoveryInspection>),
ReceiptPublishedInspectionDeferred,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoverySettlementReply {
pub outcome: DeleteRecoverySettlementOutcome,
pub visited: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BoundedDeleteRecoveryInspection {
pub inspection: DeleteRecoveryInspection,
pub visited: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub enum DeleteRecoveryRpcOperation {
SettleBounded(BoundedDeleteRecoverySettlement),
InspectBounded {
object: String,
transaction: Uuid,
max_entries: u32,
},
DiscoverBounded {
object: String,
max_entries: u32,
},
InspectOrphanDataDir {
path: String,
},
@@ -2604,11 +2447,7 @@ impl DeleteRecoveryRpcOperation {
/// Full metadata application and adoption share the existing mutation envelope.
pub fn max_wire_payload_bytes(&self) -> usize {
match self {
Self::SettleBounded(BoundedDeleteRecoverySettlement {
action: DeleteRecoveryAction::AdoptCommit { .. },
..
})
| Self::ApplyVersion { .. }
Self::ApplyVersion { .. }
| Self::ApplyVersions { .. }
| Self::Settle {
action: DeleteRecoveryAction::AdoptCommit { .. },
@@ -2624,11 +2463,8 @@ impl DeleteRecoveryRpcOperation {
pub enum DeleteRecoveryRpcReply {
OrphanDataDir { plain: bool },
Discovered(DeleteRecoveryPage),
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
Inspected(DeleteRecoveryInspection),
InspectedBounded(BoundedDeleteRecoveryInspection),
Settled(DeleteRecoveryInspection),
SettledBounded(BoundedDeleteRecoverySettlementReply),
AppliedVersion,
AppliedVersions(Vec<Option<u32>>),
Evidence { present: bool },
@@ -3106,8 +2942,6 @@ mod tests {
recursive: true,
incl_deleted: false,
skip_hidden_prefix_check: false,
hidden_entry_budget: 0,
pending_prefix: None,
report_notfound: false,
filter_prefix: Some("prefix_".to_string()),
forward_to: Some("object/path".to_string()),
-275
View File
@@ -1,275 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! A candidate index never replaces multipart metadata or quorum validation.
use serde::{Deserialize, Serialize};
use uuid::Uuid;
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
/// A bounded page from one disk's continuously maintained, boot-local index.
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
/// pages from different epochs or physical disk identities.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexPage {
pub bucket: String,
pub disk_id: Uuid,
pub epoch: Uuid,
pub roots: Vec<String>,
pub next_marker: Option<String>,
}
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
/// Old peers may ignore this field, but their ordinary response cannot decode
/// as the required signed, challenge-bound MultipartIndexRpcReply.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexRpcRequest {
pub disk_id: String,
pub metrics: bool,
pub noop: bool,
pub multipart_index: MultipartIndexQuery,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexQuery {
pub operation: String,
pub version: u32,
pub challenge: Uuid,
pub bucket: String,
pub marker: Option<String>,
pub page_cap: usize,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MultipartIndexRpcReply {
pub operation: String,
pub version: u32,
pub challenge: Uuid,
pub page: MultipartIndexPage,
pub proof: Vec<u8>,
}
impl MultipartIndexRpcRequest {
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
Self {
disk_id: String::new(),
metrics: false,
noop: true,
multipart_index: MultipartIndexQuery {
operation: "list_multipart_index_roots".into(),
version: 1,
challenge: Uuid::new_v4(),
bucket: bucket.to_owned(),
marker: marker.map(str::to_owned),
page_cap: MULTIPART_INDEX_PAGE_SIZE,
},
}
}
pub fn validate(&self) -> crate::disk::error::Result<()> {
let query = &self.multipart_index;
if !self.disk_id.is_empty()
|| self.metrics
|| !self.noop
|| query.operation != "list_multipart_index_roots"
|| query.version != 1
|| query.challenge.is_nil()
|| query.bucket.is_empty()
|| query.bucket.len() > 255
|| query.bucket.contains('/')
|| matches!(query.bucket.as_str(), "." | "..")
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
{
return Err(crate::disk::error::DiskError::MethodNotAllowed);
}
Ok(())
}
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
self.validate()?;
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
return Err(crate::disk::error::DiskError::FileCorrupt);
}
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
}
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
self.validate()?;
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
}
}
fn valid_root(root: &str) -> bool {
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
}
impl MultipartIndexPage {
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
if expected_disk.is_nil()
|| self.disk_id != expected_disk
|| self.epoch.is_nil()
|| self.bucket != bucket
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|| self
.roots
.iter()
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|| self
.next_marker
.as_ref()
.is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
{
return Err(crate::disk::error::DiskError::FileCorrupt);
}
Ok(())
}
}
impl MultipartIndexRpcReply {
/// This is structural validation only; the caller must also verify `proof`.
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
let query = &request.multipart_index;
request.validate()?;
if self.operation != query.operation
|| self.version != query.version
|| self.challenge != query.challenge
|| self.proof.len() != 32
{
return Err(crate::disk::error::DiskError::FileCorrupt);
}
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
}
}
/// A disk can attest coverage only after every page belongs to the same physical
/// identity and index generation. An interrupted/changed stream contributes no
/// partial candidates to the erasure set's coverage count.
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
where
F: FnMut(Option<String>) -> Fut,
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
{
let mut epoch = None;
let mut marker = None;
let mut roots = Vec::new();
loop {
let page = fetch(marker.clone()).await?;
page.validate(bucket, marker.as_deref(), identity)?;
if epoch.is_some_and(|epoch| epoch != page.epoch) {
return Err(crate::disk::error::DiskError::DiskNotFound);
}
epoch = Some(page.epoch);
roots.extend(page.roots);
marker = page.next_marker;
if marker.is_none() {
return Ok(roots);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
let request = MultipartIndexRpcRequest::new("bucket", None);
let id = Uuid::new_v4();
let mut reply = MultipartIndexRpcReply {
operation: request.multipart_index.operation.clone(),
version: 1,
challenge: request.multipart_index.challenge,
page: MultipartIndexPage {
bucket: "bucket".into(),
disk_id: id,
epoch: Uuid::new_v4(),
roots: vec!["a".repeat(64)],
next_marker: None,
},
proof: vec![0; 32],
};
reply.validate(&request, id).expect("valid structural envelope");
assert!(
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
"old challenge cannot replay"
);
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
reply.page.next_marker = Some("a".repeat(64));
assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
reply.page.next_marker = None;
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
}
#[test]
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
crate::runtime::sources::ensure_test_rpc_secret();
let mut request = MultipartIndexRpcRequest::new("bucket", None);
let mut page = MultipartIndexPage {
bucket: "bucket".into(),
disk_id: Uuid::new_v4(),
epoch: Uuid::new_v4(),
roots: vec![],
next_marker: None,
};
let canonical = request.canonical_response("disk", &page).expect("canonical response");
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
crate::cluster::rpc::verify_tonic_rpc_response_proof(
&request.canonical_response("disk", page).expect("canonical"),
&proof,
)
};
verify(&request, &page).expect("exact response");
request.multipart_index.marker = Some("0".repeat(64));
assert!(verify(&request, &page).is_err(), "cursor is signed");
request.multipart_index.marker = None;
page.epoch = Uuid::new_v4();
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
}
#[tokio::test]
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
let id = Uuid::new_v4();
let epoch = Uuid::new_v4();
for replace_disk in [false, true] {
let first = MultipartIndexPage {
bucket: "bucket".into(),
disk_id: id,
epoch,
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
};
let second = MultipartIndexPage {
bucket: "bucket".into(),
disk_id: if replace_disk { Uuid::new_v4() } else { id },
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
roots: vec![],
next_marker: None,
};
let mut replies = std::collections::VecDeque::from([first, second]);
let result = collect_pages("bucket", id, |marker| {
let page = replies.pop_front().expect("bounded fixture pages");
if marker.is_some() {
assert!(page.roots.is_empty());
}
std::future::ready(Ok(page))
})
.await;
assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
assert!(replies.is_empty(), "exercise the continuation response");
}
}
}
+77 -769
View File
@@ -1677,396 +1677,8 @@ const MAX_GLOBAL_FILE_SYNCS: usize = 512;
#[cfg(test)]
const TEST_GLOBAL_FILE_SYNCS: usize = 64;
const PATH_LOCK_SHARDS: usize = 64;
const PATH_LOCK_INITIAL_GC_THRESHOLD: usize = 64;
#[cfg(unix)]
#[derive(Clone, Copy, Default)]
struct Xxh3SliceHasher(u64);
#[cfg(unix)]
impl std::hash::Hasher for Xxh3SliceHasher {
#[inline]
fn write(&mut self, bytes: &[u8]) {
self.0 = xxhash_rust::xxh3::xxh3_64(bytes);
}
#[inline]
fn write_usize(&mut self, _i: usize) {}
#[inline]
fn finish(&self) -> u64 {
self.0
}
}
#[cfg(unix)]
type Xxh3SliceBuildHasher = std::hash::BuildHasherDefault<Xxh3SliceHasher>;
#[cfg(unix)]
#[inline]
fn is_canonical_unix_key_bytes(bytes: &[u8]) -> bool {
if bytes.starts_with(b"./") {
return !bytes[1..].windows(2).any(|w| w == b"//") && !bytes[1..].windows(3).any(|w| w == b"/./");
}
const SLASH_MASK: u64 = 0x2f2f_2f2f_2f2f_2f2f;
const LOW_BITS: u64 = 0x0101_0101_0101_0101;
const HIGH_BITS: u64 = 0x8080_8080_8080_8080;
let mut i = 0usize;
let len = bytes.len();
let mut prev_s = 0u64;
while i + 8 <= len {
let mut buf = [0u8; 8];
buf.copy_from_slice(&bytes[i..i + 8]);
let w = u64::from_le_bytes(buf);
let x = w ^ SLASH_MASK;
let s = (x.wrapping_sub(LOW_BITS)) & !x & HIGH_BITS;
let s_prev1 = (s << 8) | (prev_s >> 56);
let s_prev2 = (s << 16) | (prev_s >> 48);
if (s & (s_prev1 | s_prev2)) != 0 {
let start = i.saturating_sub(2);
let window = &bytes[start..i + 8];
if window.windows(2).any(|pair| pair == b"//") || window.windows(3).any(|triple| triple == b"/./") {
return false;
}
}
prev_s = s;
i += 8;
}
let tail_start = i.saturating_sub(2);
let tail = &bytes[tail_start..];
!tail.windows(2).any(|pair| pair == b"//") && !tail.windows(3).any(|triple| triple == b"/./")
}
#[cfg(unix)]
#[cold]
fn canonical_unix_path_key_slow(path: &Path) -> std::borrow::Cow<'static, [u8]> {
use std::os::unix::ffi::OsStrExt;
let mut out = Vec::with_capacity(path.as_os_str().len());
for comp in path.components() {
match comp {
Component::RootDir => {
out.clear();
out.push(b'/');
}
Component::CurDir => {
if out.is_empty() {
out.push(b'.');
}
}
Component::ParentDir => {
if out.len() > 1 || (out.len() == 1 && out[0] != b'/') {
out.push(b'/');
}
out.extend_from_slice(b"..");
}
Component::Normal(part) => {
if out.len() > 1 || (out.len() == 1 && out[0] != b'/') {
out.push(b'/');
}
out.extend_from_slice(part.as_bytes());
}
Component::Prefix(_) => {}
}
}
std::borrow::Cow::Owned(out)
}
#[cfg(unix)]
#[inline]
fn canonical_unix_path_key(path: &Path) -> std::borrow::Cow<'_, [u8]> {
use std::os::unix::ffi::OsStrExt;
let mut bytes = path.as_os_str().as_bytes();
while bytes.len() > 1 {
if bytes[bytes.len() - 1] == b'/' {
bytes = &bytes[..bytes.len() - 1];
} else if bytes.len() >= 2 && bytes[bytes.len() - 2] == b'/' && bytes[bytes.len() - 1] == b'.' {
bytes = if bytes.len() == 2 { b"/" } else { &bytes[..bytes.len() - 2] };
} else {
break;
}
}
if is_canonical_unix_key_bytes(bytes) {
return std::borrow::Cow::Borrowed(bytes);
}
canonical_unix_path_key_slow(path)
}
#[cfg(unix)]
#[inline]
fn unix_canonical_key_starts_with(path_key: &[u8], root_key: &[u8]) -> bool {
if root_key.is_empty() {
return true;
}
if root_key == b"/" {
return path_key.starts_with(b"/");
}
path_key.starts_with(root_key) && (path_key.len() == root_key.len() || path_key.get(root_key.len()) == Some(&b'/'))
}
struct PathLockShard<L> {
#[cfg(unix)]
entries: HashMap<Box<[u8]>, Weak<L>, Xxh3SliceBuildHasher>,
#[cfg(not(unix))]
entries: HashMap<PathBuf, Weak<L>>,
next_gc_threshold: usize,
}
impl<L> Default for PathLockShard<L> {
fn default() -> Self {
Self {
entries: HashMap::default(),
next_gc_threshold: PATH_LOCK_INITIAL_GC_THRESHOLD,
}
}
}
struct ShardedPathLockRegistry<L> {
shards: [Mutex<PathLockShard<L>>; PATH_LOCK_SHARDS],
}
impl<L> ShardedPathLockRegistry<L> {
fn new() -> Self {
Self {
shards: std::array::from_fn(|_| Mutex::new(PathLockShard::default())),
}
}
#[cfg(unix)]
#[inline]
fn get_or_create_by_key(&self, key: &[u8], init: impl FnOnce() -> L) -> Arc<L> {
let hash = xxhash_rust::xxh3::xxh3_64(key);
let shard_idx = ((hash >> 32) as usize) & (PATH_LOCK_SHARDS - 1);
let mut shard = self.shards[shard_idx].lock();
if let Some(existing) = shard.entries.get(key).and_then(Weak::upgrade) {
return existing;
}
let created = Arc::new(init());
let weak = Arc::downgrade(&created);
if let Some(slot) = shard.entries.get_mut(key) {
*slot = weak;
return created;
}
if shard.entries.len() >= shard.next_gc_threshold {
shard.entries.retain(|_, entry| entry.strong_count() > 0);
shard.next_gc_threshold = shard
.entries
.len()
.saturating_add(1)
.saturating_mul(2)
.max(PATH_LOCK_INITIAL_GC_THRESHOLD);
}
shard.entries.insert(key.into(), weak);
created
}
#[inline]
fn get_or_create(&self, path: &Path, init: impl FnOnce() -> L) -> Arc<L> {
#[cfg(unix)]
{
let key = canonical_unix_path_key(path);
self.get_or_create_by_key(key.as_ref(), init)
}
#[cfg(not(unix))]
{
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
path.hash(&mut hasher);
let shard_idx = ((hasher.finish() >> 32) as usize) & (PATH_LOCK_SHARDS - 1);
let mut shard = self.shards[shard_idx].lock();
if let Some(existing) = shard.entries.get(path).and_then(Weak::upgrade) {
return existing;
}
let created = Arc::new(init());
let weak = Arc::downgrade(&created);
if let Some(slot) = shard.entries.get_mut(path) {
*slot = weak;
return created;
}
if shard.entries.len() >= shard.next_gc_threshold {
shard.entries.retain(|_, entry| entry.strong_count() > 0);
shard.next_gc_threshold = shard
.entries
.len()
.saturating_add(1)
.saturating_mul(2)
.max(PATH_LOCK_INITIAL_GC_THRESHOLD);
}
shard.entries.insert(path.to_path_buf(), weak);
created
}
}
}
#[cfg(unix)]
type PathCacheShardMap<V> = HashMap<Box<[u8]>, V, Xxh3SliceBuildHasher>;
#[cfg(not(unix))]
type PathCacheShardMap<V> = HashMap<PathBuf, V>;
pub(crate) struct ShardedPathCache<V> {
shards: [Mutex<PathCacheShardMap<V>>; PATH_LOCK_SHARDS],
max_entries_per_shard: usize,
}
impl<V> ShardedPathCache<V> {
pub(crate) fn new(max_entries_per_shard: usize) -> Self {
Self {
shards: std::array::from_fn(|_| Mutex::new(HashMap::default())),
max_entries_per_shard: max_entries_per_shard.max(1),
}
}
#[cfg(unix)]
#[inline]
fn shard_index(key: &[u8]) -> usize {
let hash = xxhash_rust::xxh3::xxh3_64(key);
((hash >> 32) as usize) & (PATH_LOCK_SHARDS - 1)
}
#[cfg(not(unix))]
#[inline]
fn shard_index(path: &Path) -> usize {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
path.hash(&mut hasher);
((hasher.finish() >> 32) as usize) & (PATH_LOCK_SHARDS - 1)
}
#[inline]
pub(crate) fn insert(&self, path: &Path, value: V) {
#[cfg(unix)]
{
let key = canonical_unix_path_key(path);
let shard_idx = Self::shard_index(key.as_ref());
let mut shard = self.shards[shard_idx].lock();
if let Some(slot) = shard.get_mut(key.as_ref()) {
*slot = value;
return;
}
if shard.len() >= self.max_entries_per_shard
&& let Some(evict_key) = shard.keys().next().cloned()
{
shard.remove(&evict_key);
}
shard.insert(key.into_owned().into_boxed_slice(), value);
}
#[cfg(not(unix))]
{
let shard_idx = Self::shard_index(path);
let mut shard = self.shards[shard_idx].lock();
if let Some(slot) = shard.get_mut(path) {
*slot = value;
return;
}
if shard.len() >= self.max_entries_per_shard
&& let Some(evict_key) = shard.keys().next().cloned()
{
shard.remove(&evict_key);
}
shard.insert(path.to_path_buf(), value);
}
}
#[inline]
pub(crate) fn remove(&self, path: &Path) -> Option<V> {
#[cfg(unix)]
{
let key = canonical_unix_path_key(path);
let shard_idx = Self::shard_index(key.as_ref());
self.shards[shard_idx].lock().remove(key.as_ref())
}
#[cfg(not(unix))]
{
let shard_idx = Self::shard_index(path);
self.shards[shard_idx].lock().remove(path)
}
}
/// Runs `f` while holding the target shard mutex. `f` must not block, perform I/O, or re-enter `self`.
#[inline]
pub(crate) fn with_get<R>(&self, path: &Path, f: impl FnOnce(&V) -> R) -> Option<R> {
#[cfg(unix)]
{
let key = canonical_unix_path_key(path);
let shard_idx = Self::shard_index(key.as_ref());
let shard = self.shards[shard_idx].lock();
shard.get(key.as_ref()).map(f)
}
#[cfg(not(unix))]
{
let shard_idx = Self::shard_index(path);
let shard = self.shards[shard_idx].lock();
shard.get(path).map(f)
}
}
/// Runs `f` while holding the target shard mutex. `f` must not block, perform I/O, or re-enter `self`.
#[inline]
pub(crate) fn with_get_mut<R>(&self, path: &Path, f: impl FnOnce(&mut V) -> R) -> Option<R> {
#[cfg(unix)]
{
let key = canonical_unix_path_key(path);
let shard_idx = Self::shard_index(key.as_ref());
let mut shard = self.shards[shard_idx].lock();
shard.get_mut(key.as_ref()).map(f)
}
#[cfg(not(unix))]
{
let shard_idx = Self::shard_index(path);
let mut shard = self.shards[shard_idx].lock();
shard.get_mut(path).map(f)
}
}
/// Runs `should_remove` while holding the target shard mutex. `should_remove` must not block, perform I/O, or re-enter `self`.
#[inline]
pub(crate) fn remove_if_mut(&self, path: &Path, should_remove: impl FnOnce(&mut V) -> bool) {
#[cfg(unix)]
{
let key = canonical_unix_path_key(path);
let shard_idx = Self::shard_index(key.as_ref());
let mut shard = self.shards[shard_idx].lock();
if let Some(entry) = shard.get_mut(key.as_ref())
&& should_remove(entry)
{
shard.remove(key.as_ref());
}
}
#[cfg(not(unix))]
{
let shard_idx = Self::shard_index(path);
let mut shard = self.shards[shard_idx].lock();
if let Some(entry) = shard.get_mut(path)
&& should_remove(entry)
{
shard.remove(path);
}
}
}
pub(crate) fn retain_outside_prefix(&self, prefix: &Path) {
#[cfg(unix)]
{
let prefix_key = canonical_unix_path_key(prefix);
let prefix_bytes = prefix_key.as_ref();
for shard in &self.shards {
shard
.lock()
.retain(|key, _| !unix_canonical_key_starts_with(key, prefix_bytes));
}
}
#[cfg(not(unix))]
{
for shard in &self.shards {
shard.lock().retain(|key, _| !key.starts_with(prefix));
}
}
}
}
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<ShardedPathLockRegistry<Semaphore>> = LazyLock::new(ShardedPathLockRegistry::new);
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
/// configured with >1 threads, isolates device-bound fsync from the main
@@ -2130,10 +1742,12 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
f()
})
}
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<RwLock<()>>> = LazyLock::new(ShardedPathLockRegistry::new);
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
type NamespaceMutationLock = AsyncMutex<()>;
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<NamespaceMutationLock>> =
LazyLock::new(ShardedPathLockRegistry::new);
type NamespaceMutationLockRegistry = HashMap<PathBuf, Weak<NamespaceMutationLock>>;
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
let cpu_scaled = cpu_count
@@ -2157,7 +1771,15 @@ fn global_file_sync_limit() -> usize {
/// Reuse a disk's limiter across reconnects while detached sync calls still hold it.
pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
DISK_FILE_SYNC_LIMITERS.get_or_create(root, || Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
let mut limiters = DISK_FILE_SYNC_LIMITERS.lock();
limiters.retain(|_, limiter| limiter.strong_count() > 0);
if let Some(limiter) = limiters.get(root).and_then(Weak::upgrade) {
return limiter;
}
let limiter = Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS));
limiters.insert(root.to_path_buf(), Arc::downgrade(&limiter));
limiter
}
// Keep warm draining workers per disk/lane and spin briefly in user space
@@ -2525,131 +2147,45 @@ pub(crate) fn enter_blocking_test_hook() -> BlockingTestHookGuard {
BlockingTestHookGuard { queue }
}
struct RegisteredDiskBlockingQueue {
#[cfg(not(unix))]
root: PathBuf,
#[cfg(unix)]
canonical_root: Box<[u8]>,
queue: Weak<DiskBlockingQueue>,
}
static DISK_BLOCKING_QUEUES: LazyLock<parking_lot::RwLock<Vec<RegisteredDiskBlockingQueue>>> =
LazyLock::new(|| parking_lot::RwLock::new(Vec::new()));
static DISK_BLOCKING_QUEUES: LazyLock<Mutex<HashMap<PathBuf, Weak<DiskBlockingQueue>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
static FALLBACK_BLOCKING_QUEUES: LazyLock<[Arc<DiskBlockingQueue>; FALLBACK_NAMESPACE_QUEUE_SHARDS]> =
LazyLock::new(|| std::array::from_fn(|_| DiskBlockingQueue::new()));
pub(crate) fn register_disk_blocking_queue(root: &Path, io_root: &Path) -> Arc<DiskBlockingQueue> {
#[cfg(unix)]
let root_key = canonical_unix_path_key(root);
#[cfg(unix)]
let io_root_key = canonical_unix_path_key(io_root);
let mut queues = DISK_BLOCKING_QUEUES.write();
queues.retain(|entry| entry.queue.strong_count() > 0);
let queue = queues
.iter()
.find_map(|entry| {
#[cfg(unix)]
let matches = entry.canonical_root.as_ref() == root_key.as_ref();
#[cfg(not(unix))]
let matches = entry.root == root;
matches.then(|| entry.queue.upgrade()).flatten()
})
.unwrap_or_else(|| {
let queue = DiskBlockingQueue::new();
queues.push(RegisteredDiskBlockingQueue {
#[cfg(not(unix))]
root: root.to_path_buf(),
#[cfg(unix)]
canonical_root: root_key.as_ref().into(),
queue: Arc::downgrade(&queue),
});
queue
});
#[cfg(unix)]
let io_matches_root = io_root_key.as_ref() == root_key.as_ref();
#[cfg(not(unix))]
let io_matches_root = io_root == root;
if !io_matches_root {
let weak = Arc::downgrade(&queue);
if let Some(existing) = queues.iter_mut().find(|entry| {
#[cfg(unix)]
{
entry.canonical_root.as_ref() == io_root_key.as_ref()
}
#[cfg(not(unix))]
{
entry.root == io_root
}
}) {
existing.queue = weak;
} else {
queues.push(RegisteredDiskBlockingQueue {
#[cfg(not(unix))]
root: io_root.to_path_buf(),
#[cfg(unix)]
canonical_root: io_root_key.as_ref().into(),
queue: weak,
});
}
let mut queues = DISK_BLOCKING_QUEUES.lock();
queues.retain(|_, q| q.strong_count() > 0);
let queue = queues.get(root).and_then(Weak::upgrade).unwrap_or_else(|| {
let queue = DiskBlockingQueue::new();
queues.insert(root.to_path_buf(), Arc::downgrade(&queue));
queue
});
if io_root != root {
queues.insert(io_root.to_path_buf(), Arc::downgrade(&queue));
}
queue
}
fn disk_blocking_queue_for_root(root: &Path) -> Arc<DiskBlockingQueue> {
#[cfg(unix)]
let root_key = canonical_unix_path_key(root);
{
let queues = DISK_BLOCKING_QUEUES.read();
for entry in queues.iter() {
#[cfg(unix)]
let matches = entry.canonical_root.as_ref() == root_key.as_ref();
#[cfg(not(unix))]
let matches = entry.root == root;
if matches && let Some(queue) = entry.queue.upgrade() {
return queue;
}
}
}
register_disk_blocking_queue(root, root)
}
fn disk_blocking_queue_for_path(path: &Path) -> Arc<DiskBlockingQueue> {
#[cfg(unix)]
{
let path_key = canonical_unix_path_key(path);
{
let queues = DISK_BLOCKING_QUEUES.read();
for entry in queues.iter() {
if unix_canonical_key_starts_with(path_key.as_ref(), entry.canonical_root.as_ref())
&& let Some(queue) = entry.queue.upgrade()
{
return queue;
}
let queues = DISK_BLOCKING_QUEUES.lock();
for (root, weak) in queues.iter() {
if path.starts_with(root)
&& let Some(queue) = weak.upgrade()
{
return queue;
}
}
let hash = xxhash_rust::xxh3::xxh3_64(path_key.as_ref());
let idx = usize::try_from(hash % (FALLBACK_NAMESPACE_QUEUE_SHARDS as u64)).unwrap_or(0);
Arc::clone(&FALLBACK_BLOCKING_QUEUES[idx])
}
#[cfg(not(unix))]
{
{
let queues = DISK_BLOCKING_QUEUES.read();
for entry in queues.iter() {
if path.starts_with(&entry.root)
&& let Some(queue) = entry.queue.upgrade()
{
return queue;
}
}
}
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
path.hash(&mut hasher);
let idx = usize::try_from(hasher.finish() % (FALLBACK_NAMESPACE_QUEUE_SHARDS as u64)).unwrap_or(0);
Arc::clone(&FALLBACK_BLOCKING_QUEUES[idx])
}
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
path.hash(&mut hasher);
let idx = usize::try_from(hasher.finish() % (FALLBACK_NAMESPACE_QUEUE_SHARDS as u64)).unwrap_or(0);
Arc::clone(&FALLBACK_BLOCKING_QUEUES[idx])
}
/// Serialize a bucket's local metadata commits with physical bucket removal.
@@ -2658,46 +2194,32 @@ fn disk_blocking_queue_for_path(path: &Path) -> Arc<DiskBlockingQueue> {
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
/// disconnected disks do not keep the registry alive.
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
#[cfg(unix)]
{
let vol_bytes = volume.as_bytes();
if !vol_bytes.is_empty()
&& vol_bytes[0] != b'/'
&& vol_bytes != b"."
&& !vol_bytes.starts_with(b"./")
&& !vol_bytes.ends_with(b"/")
&& !vol_bytes.ends_with(b"/.")
&& is_canonical_unix_key_bytes(vol_bytes)
&& let std::borrow::Cow::Borrowed(root_bytes) = canonical_unix_path_key(root)
&& !root_bytes.is_empty()
&& root_bytes != b"/"
{
let total_len = root_bytes.len() + 1 + vol_bytes.len();
if total_len <= 512 {
let mut stack_key = [0u8; 512];
stack_key[..root_bytes.len()].copy_from_slice(root_bytes);
stack_key[root_bytes.len()] = b'/';
stack_key[root_bytes.len() + 1..total_len].copy_from_slice(vol_bytes);
return DISK_VOLUME_MUTATION_LOCKS.get_or_create_by_key(&stack_key[..total_len], || RwLock::new(()));
}
}
}
let key = root.join(volume);
DISK_VOLUME_MUTATION_LOCKS.get_or_create(&key, || RwLock::new(()))
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
return lock;
}
let lock = Arc::new(RwLock::new(()));
locks.insert(key, Arc::downgrade(&lock));
lock
}
fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
DISK_NAMESPACE_MUTATION_LOCKS.get_or_create(path, || AsyncMutex::new(()))
let mut locks = DISK_NAMESPACE_MUTATION_LOCKS.lock();
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(path).and_then(Weak::upgrade) {
return lock;
}
let lock = Arc::new(AsyncMutex::new(()));
locks.insert(path.to_path_buf(), Arc::downgrade(&lock));
lock
}
#[cfg(unix)]
#[inline]
fn disk_namespace_mutation_lock_by_key(key: &[u8]) -> Arc<NamespaceMutationLock> {
DISK_NAMESPACE_MUTATION_LOCKS.get_or_create_by_key(key, || AsyncMutex::new(()))
}
static DISK_METADATA_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<NamespaceMutationLock>> =
LazyLock::new(ShardedPathLockRegistry::new);
static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
/// Serializes the complete xl.meta read/modify/commit transaction. This domain
/// precedes namespace/volume publication locks, whose narrower syscall leases
@@ -2711,7 +2233,17 @@ pub(crate) async fn acquire_metadata_mutation_lease(
object: &Path,
owner: Option<Arc<dyn Send + Sync>>,
) -> Arc<MetadataMutationLease> {
let lock = DISK_METADATA_MUTATION_LOCKS.get_or_create(object, || AsyncMutex::new(()));
let lock = {
let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
lock
} else {
let lock = Arc::new(AsyncMutex::new(()));
locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
lock
}
};
Arc::new(MetadataMutationLease {
_guard: lock.lock_owned().await,
_owner: owner,
@@ -2731,11 +2263,6 @@ pub(crate) struct NamespaceMutationLease {
}
impl NamespaceMutationLease {
#[cfg(test)]
pub(crate) fn blocking_queue_for_test(&self) -> Arc<DiskBlockingQueue> {
self.blocking_queue.clone()
}
pub(crate) fn attach_external_guard(&self, guard: Arc<dyn Send + Sync>) {
*self.external_guard.lock() = Some(guard);
}
@@ -2779,45 +2306,17 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
) -> Arc<NamespaceMutationLease> {
// Metadata ownership precedes namespace paths in lexical order, then volume
// ownership. Deduplicate identical keys so TMP-to-TMP publication cannot self-lock.
#[cfg(unix)]
let (namespace_guard, staging_guard) = {
let dst_key = canonical_unix_path_key(destination_object);
let destination_lock = disk_namespace_mutation_lock_by_key(dst_key.as_ref());
match staging_prefix {
Some(source) => {
let src_key = canonical_unix_path_key(source);
match src_key.as_ref().cmp(dst_key.as_ref()) {
std::cmp::Ordering::Less => {
let staging = disk_namespace_mutation_lock_by_key(src_key.as_ref()).lock_owned().await;
(destination_lock.lock_owned().await, Some(staging))
}
std::cmp::Ordering::Greater => {
let destination = destination_lock.lock_owned().await;
(
destination,
Some(disk_namespace_mutation_lock_by_key(src_key.as_ref()).lock_owned().await),
)
}
std::cmp::Ordering::Equal => (destination_lock.lock_owned().await, None),
}
}
None => (destination_lock.lock_owned().await, None),
let destination_lock = disk_namespace_mutation_lock(destination_object);
let (namespace_guard, staging_guard) = match staging_prefix {
Some(source) if source < destination_object => {
let staging = disk_namespace_mutation_lock(source).lock_owned().await;
(destination_lock.lock_owned().await, Some(staging))
}
};
#[cfg(not(unix))]
let (namespace_guard, staging_guard) = {
let destination_lock = disk_namespace_mutation_lock(destination_object);
match staging_prefix {
Some(source) if source < destination_object => {
let staging = disk_namespace_mutation_lock(source).lock_owned().await;
(destination_lock.lock_owned().await, Some(staging))
}
Some(source) if source > destination_object => {
let destination = destination_lock.lock_owned().await;
(destination, Some(disk_namespace_mutation_lock(source).lock_owned().await))
}
_ => (destination_lock.lock_owned().await, None),
Some(source) if source > destination_object => {
let destination = destination_lock.lock_owned().await;
(destination, Some(disk_namespace_mutation_lock(source).lock_owned().await))
}
_ => (destination_lock.lock_owned().await, None),
};
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
let blocking_queue = disk_blocking_queue_for_root(root);
@@ -5387,29 +4886,6 @@ impl PublicationRoot {
})
}
/// Distinguish reconnects from replacement mounts before reusing disk-local state.
pub(crate) fn physical_identity(&self) -> io::Result<(u64, [u8; 16])> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
#[cfg(target_os = "linux")]
let metadata = self.directory.metadata()?;
#[cfg(not(target_os = "linux"))]
let metadata = std::fs::metadata(&*self.path)?;
let mut inode = [0; 16];
inode[..8].copy_from_slice(&metadata.ino().to_le_bytes());
Ok((metadata.dev(), inode))
}
#[cfg(windows)]
{
windows_file_identity(&self.directory.handle)
}
#[cfg(not(any(unix, windows)))]
{
Err(io::Error::new(io::ErrorKind::Unsupported, "physical root identity unavailable"))
}
}
pub(crate) fn path(&self) -> &Path {
&self.path
}
@@ -12516,174 +11992,6 @@ mod tests {
);
assert!(preserved_final.exists(), "final file must remain when intermediate removal fails");
}
#[cfg(unix)]
#[test]
fn canonical_unix_path_key_preserves_std_path_equality_and_prefix_semantics() {
let samples = [
"",
"/",
"///",
"/.",
"/./.",
".",
"./",
"./.",
"./a",
"././a/",
".//a/.",
"a",
"a/",
"a/.",
"a/b/c",
"a/b/c/",
"a//b/./c/.",
"/tmp/d0/bucket/obj",
"/tmp/d0/bucket/obj/",
"/tmp/d0//bucket/./obj/.",
"/tmp/d0/.rustfs.sys/multipart/sha/upload",
"/proc/self/fd/12/.",
"/proc/self/fd/12",
"/proc/self/fd/12/./bucket/obj",
"/proc/self/fd/12/bucket/obj",
"/tmp/d1",
"/tmp/d10",
"a/../b",
"a/../b/",
"/utf8/\u{00af}/segment/",
];
for s in samples {
let p = Path::new(s);
let fast = canonical_unix_path_key(p);
let slow = canonical_unix_path_key_slow(p);
assert_eq!(fast.as_ref(), slow.as_ref(), "fast and slow key mismatch for {s:?}");
}
for a in samples {
for b in samples {
let pa = Path::new(a);
let pb = Path::new(b);
let ka = canonical_unix_path_key(pa);
let kb = canonical_unix_path_key(pb);
assert_eq!(
pa == pb,
ka.as_ref() == kb.as_ref(),
"Path::eq vs canonical key mismatch for {a:?} and {b:?}"
);
assert_eq!(
pa.starts_with(pb),
unix_canonical_key_starts_with(ka.as_ref(), kb.as_ref()),
"Path::starts_with vs canonical key prefix mismatch for {a:?} and {b:?}"
);
}
}
let dir = tempdir().expect("tempdir");
let p_clean = dir.path().join("bucket/object");
let p_trailing = PathBuf::from(format!("{}/", p_clean.display()));
let p_redundant = PathBuf::from(format!("{}//bucket/./object/.", dir.path().display()));
let lock1 = disk_namespace_mutation_lock(&p_clean);
let lock2 = disk_namespace_mutation_lock(&p_trailing);
let lock3 = disk_namespace_mutation_lock(&p_redundant);
assert!(Arc::ptr_eq(&lock1, &lock2));
assert!(Arc::ptr_eq(&lock1, &lock3));
let vlock1 = disk_volume_mutation_lock(dir.path(), "bucket");
let vlock2 = disk_volume_mutation_lock(&PathBuf::from(format!("{}/.", dir.path().display())), "bucket/");
assert!(Arc::ptr_eq(&vlock1, &vlock2));
let io_root = dir.path().join("io_root/.");
let q_reg = register_disk_blocking_queue(dir.path(), &io_root);
let q_root = disk_blocking_queue_for_root(dir.path());
let q_path = disk_blocking_queue_for_path(&dir.path().join("io_root/bucket/key"));
assert!(Arc::ptr_eq(&q_reg, &q_root));
assert!(Arc::ptr_eq(&q_reg, &q_path));
}
#[test]
fn sharded_path_lock_registry_reuses_slots_and_bounds_dead_weak_entries() {
let registry: ShardedPathLockRegistry<AsyncMutex<()>> = ShardedPathLockRegistry::new();
let target = Path::new("/tmp/sharded-lock-test/persistent-key");
let held = registry.get_or_create(target, || AsyncMutex::new(()));
let held_again = registry.get_or_create(target, || AsyncMutex::new(()));
assert!(Arc::ptr_eq(&held, &held_again), "live lock must be shared");
// Re-acquiring a key after drop must reuse the existing slot in-place.
let transient = Path::new("/tmp/sharded-lock-test/reused-slot");
drop(registry.get_or_create(transient, || AsyncMutex::new(())));
let reused = registry.get_or_create(transient, || AsyncMutex::new(()));
drop(reused);
// Flood transient keys across shards and verify `held` is never evicted
// and total dead entries remain bounded by the amortized GC threshold.
for i in 0..(PATH_LOCK_SHARDS * PATH_LOCK_INITIAL_GC_THRESHOLD * 3) {
let p = PathBuf::from(format!("/tmp/sharded-lock-test/ephemeral-{i}"));
drop(registry.get_or_create(&p, || AsyncMutex::new(())));
}
let still_held = registry.get_or_create(target, || AsyncMutex::new(()));
assert!(Arc::ptr_eq(&held, &still_held), "GC must never evict a live lock");
for shard in &registry.shards {
let guard = shard.lock();
assert!(
guard.entries.len() <= guard.next_gc_threshold,
"shard entries ({}) exceeded GC threshold ({})",
guard.entries.len(),
guard.next_gc_threshold
);
}
}
#[test]
fn sharded_path_cache_normalizes_keys_reuses_slots_and_filters_prefixes() {
let cache: ShardedPathCache<usize> = ShardedPathCache::new(2);
let clean = Path::new("/tmp/sharded-cache-test/bucket/obj");
let non_canonical = Path::new("/tmp/sharded-cache-test//bucket/./obj/.");
cache.insert(clean, 10);
assert_eq!(cache.with_get(non_canonical, |v| *v), Some(10));
cache.with_get_mut(non_canonical, |v| *v += 5);
assert_eq!(cache.with_get(clean, |v| *v), Some(15));
cache.remove_if_mut(clean, |v| *v == 99);
assert_eq!(cache.with_get(clean, |v| *v), Some(15));
cache.remove_if_mut(non_canonical, |v| *v == 15);
assert_eq!(cache.with_get(clean, |v| *v), None);
cache.insert(Path::new("/tmp/sharded-cache-test/bucket/a"), 1);
cache.insert(Path::new("/tmp/sharded-cache-test/bucket/b"), 2);
cache.insert(Path::new("/tmp/sharded-cache-test/bucket-other/c"), 3);
cache.retain_outside_prefix(Path::new("/tmp/sharded-cache-test//bucket/."));
assert_eq!(cache.with_get(Path::new("/tmp/sharded-cache-test/bucket/a"), |v| *v), None);
assert_eq!(cache.with_get(Path::new("/tmp/sharded-cache-test/bucket/b"), |v| *v), None);
assert_eq!(cache.with_get(Path::new("/tmp/sharded-cache-test/bucket-other/c"), |v| *v), Some(3));
for i in 0..512 {
cache.insert(&PathBuf::from(format!("/tmp/sharded-cache-test/cap/{i}")), i);
}
for shard in &cache.shards {
assert!(shard.lock().len() <= 2);
}
#[cfg(unix)]
{
let overwrite_cache: ShardedPathCache<usize> = ShardedPathCache::new(2);
let target_shard = ShardedPathCache::<usize>::shard_index(b"/tmp/sharded-cache-test/same-0");
let mut same_shard_paths = vec![PathBuf::from("/tmp/sharded-cache-test/same-0")];
let mut candidate = 1usize;
while same_shard_paths.len() < 2 {
let p = PathBuf::from(format!("/tmp/sharded-cache-test/same-{candidate}"));
if ShardedPathCache::<usize>::shard_index(canonical_unix_path_key(&p).as_ref()) == target_shard {
same_shard_paths.push(p);
}
candidate += 1;
}
overwrite_cache.insert(&same_shard_paths[0], 100);
overwrite_cache.insert(&same_shard_paths[1], 200);
overwrite_cache.insert(&same_shard_paths[0], 101);
assert_eq!(overwrite_cache.with_get(&same_shard_paths[0], |v| *v), Some(101));
assert_eq!(overwrite_cache.with_get(&same_shard_paths[1], |v| *v), Some(200));
}
}
}
#[cfg(all(test, target_os = "linux"))]
-10
View File
@@ -269,11 +269,6 @@ pub enum StorageError {
/// retained for operator inspection.
#[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
DeleteRecoveryLegacyAmbiguous,
/// Delete recovery exhausted the per-request work budget before it could
/// classify every residue. Retrying later makes progress, so callers treat
/// it like `SlowDown`; it stays distinct so the disk error round-trips.
#[error("delete recovery work limit reached; please retry")]
DeleteRecoveryWorkLimit,
#[error("Decommission not started")]
DecommissionNotStarted,
#[error("Decommission already running")]
@@ -485,7 +480,6 @@ impl From<DiskError> for StorageError {
DiskError::InvalidPath => StorageError::InvalidPath,
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
}
}
@@ -513,7 +507,6 @@ impl StorageError {
StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
StorageError::StorageFull => DiskError::DiskFull,
StorageError::SlowDown => DiskError::TooManyOpenFiles,
StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
StorageError::InconsistentDisk => DiskError::InconsistentDisk,
StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
@@ -717,7 +710,6 @@ impl Clone for StorageError {
StorageError::StorageFull => StorageError::StorageFull,
StorageError::SlowDown => StorageError::SlowDown,
StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
StorageError::InvalidUploadIDKeyCombination(a, b) => {
StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
@@ -860,7 +852,6 @@ impl StorageError {
StorageError::StorageFull => StorageErrorCode::StorageFull,
StorageError::SlowDown => StorageErrorCode::SlowDown,
StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
@@ -967,7 +958,6 @@ impl StorageError {
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
StorageErrorCode::PrefixAccessDenied => {
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
}
+4 -62
View File
@@ -79,14 +79,6 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
}
/// Validate that replicated compression metadata can be read by this build.
///
/// # Errors
/// Returns an error for an unsupported compression scheme.
pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
compression_scheme_to_read_plan(scheme).map(|_| ())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionBackend {
Legacy,
@@ -452,12 +444,10 @@ impl WritePlan {
if let Some(encryption) = self.encryption {
let etag_nonce = match encryption.mode {
WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
// Each part write needs an independent fingerprint, including a
// retry at the same part number. This salt affects only the ETag
// MAC; encryption framing and its data nonce stay unchanged.
WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
base_nonce
}
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
};
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
.with_inner_etag_delegation(delegate_inner_etag);
@@ -568,54 +558,6 @@ mod tests {
}
}
#[tokio::test]
async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
let plaintext = b"same confidential part";
let size = i64::try_from(plaintext.len()).expect("test length");
for object_key_mode in [false, true] {
let mut tags = std::collections::HashSet::new();
for part_number in [1, 2, 1] {
let encryption = if object_key_mode {
WriteEncryption::multipart_object_key([7; 32], part_number)
} else {
WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
};
let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
let mut reader = WritePlan::new()
.with_encryption(encryption)
.apply(source, size)
.expect("part plan");
let mut ciphertext = Vec::new();
reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
let tag = reader.try_resolve_etag().expect("part fingerprint");
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
assert!(
tags.insert(tag),
"different part numbers and retries must not identify matching plaintext"
);
#[cfg(feature = "rio-v2")]
let mut decrypt = if object_key_mode {
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
} else {
DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
};
#[cfg(not(feature = "rio-v2"))]
let mut decrypt = DecryptReader::new_multipart(
Cursor::new(ciphertext),
[7; 32],
if object_key_mode { [0; 12] } else { [9; 12] },
vec![part_number as usize],
);
let mut decoded = Vec::new();
decrypt
.read_to_end(&mut decoded)
.await
.expect("existing data nonce still decrypts");
assert_eq!(decoded, plaintext);
}
}
}
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
let plaintext = b"checksum-context-through-write-plan".repeat(256);
let actual_size = plaintext.len() as i64;
-88
View File
@@ -949,81 +949,8 @@ pub enum ShardIntegrityWriteMode {
Protected,
}
/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
/// Credits cover directory entries (including a minimum charge for an empty probe) and
/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
#[derive(Debug)]
pub struct DeleteRecoveryWorkBudget {
remaining: std::sync::atomic::AtomicU32,
}
/// Entries one disk may visit per recovery step (discovery plus inspection).
/// Every disk of every set a request fans out to is admitted up to this much
/// and charged on completion, so a budget must cover that many disks.
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
impl Default for DeleteRecoveryWorkBudget {
/// Covers a full 16-drive set with headroom; `for_disks` scales it when a
/// request spans more drives than that.
fn default() -> Self {
Self::new(16 * 1024 * 1024)
}
}
impl DeleteRecoveryWorkBudget {
/// A per-request budget sized for `disks` concurrent up-front reservations.
pub(crate) fn for_disks(disks: usize) -> Self {
let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
let default = u64::from(Self::default().remaining());
Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
}
pub(crate) fn new(entries: u32) -> Self {
Self {
remaining: std::sync::atomic::AtomicU32::new(entries),
}
}
pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
use std::sync::atomic::Ordering;
self.remaining
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
(remaining > 0).then(|| remaining.saturating_sub(maximum))
})
.map(|remaining| remaining.min(maximum))
.map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
}
pub(crate) fn remaining(&self) -> u32 {
self.remaining.load(std::sync::atomic::Ordering::Acquire)
}
/// Admit one bounded disk call: refuse once the budget is spent, otherwise
/// cap the call at `maximum` or what remains. The work actually performed is
/// charged on completion with `charge`, so concurrent calls never exhaust
/// the budget by reservation alone (a bulk delete fans out dozens of
/// inspections per key); each call is still bounded by its own cap.
pub(crate) fn admit(&self, maximum: u32) -> crate::disk::error::Result<u32> {
match self.remaining() {
0 => Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit),
remaining => Ok(remaining.min(maximum)),
}
}
/// Record the entries a bounded call visited (at least one per call).
pub(crate) fn charge(&self, visited: u32) {
use std::sync::atomic::Ordering;
let _ = self.remaining.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
Some(remaining.saturating_sub(visited.max(1)))
});
}
}
#[derive(Default, Clone)]
pub struct ObjectOptions {
#[doc(hidden)]
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
// Use the maximum parity (N/2), used when saving server configuration files
pub max_parity: bool,
pub mod_time: Option<OffsetDateTime>,
@@ -1175,12 +1102,6 @@ pub struct ObjectOptions {
pub expiration: ExpirationOptions,
pub lifecycle_audit_event: LcAuditEvent,
/// In-process request gate evaluated on authoritative completion metadata.
/// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
/// implementations may own customer encryption keys.
#[doc(hidden)]
pub multipart_completion_authorization:
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
pub eval_metadata: Option<HashMap<String, String>>,
/// Internal compare-and-set condition for replication workers publishing
/// terminal status after remote I/O. Storage validates it while holding
@@ -1194,10 +1115,6 @@ pub struct ObjectOptions {
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
/// before a destructive commit reaches the set layer.
pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
/// ECStore single PUTs acquire their destructive-write configuration guards
/// only after staging the client body. External lock owners supply a snapshot.
#[doc(hidden)]
pub defer_put_object_lock_snapshot: bool,
pub want_checksum: Option<Checksum>,
pub skip_verify_bitrot: bool,
@@ -1229,11 +1146,6 @@ pub enum ReplicationStatusWritebackMode {
}
impl ObjectOptions {
pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
self.delete_recovery_work_budget
.get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
}
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
match self.shard_integrity_write_mode {
Some(ShardIntegrityWriteMode::Protected) => true,
@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
bytes.len()
};
Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
}
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
+4 -3
View File
@@ -37,6 +37,7 @@ use std::{
time::Duration,
};
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::{
select,
sync::{Notify, RwLock},
@@ -8182,7 +8183,7 @@ mod tests {
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
}
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
@@ -8247,7 +8248,7 @@ mod tests {
_opts: WarmBackendGetOpts,
) -> std::result::Result<ReadCloser, std::io::Error> {
if self.healthy {
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
} else {
Err(std::io::Error::other("mock get failed"))
}
@@ -9894,7 +9895,7 @@ mod tests {
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
}
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
+1 -8
View File
@@ -195,14 +195,7 @@ impl SetDisks {
metadata.add_version(selected)?;
}
if metadata.versions.is_empty() {
// With no decodable version, the per-version quorum loop cannot
// reject corrupt or unavailable replicas. Only complete, readable
// evidence may establish that this key has no authoritative version.
return if complete {
Ok(None)
} else {
Err(DiskError::ErasureReadQuorum)
};
return Ok(None);
}
Ok(Some(MetaCacheEntry {
name,
+8 -32
View File
@@ -357,7 +357,8 @@ use crate::io_support::rio::HashReader;
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
pub const MAX_PARTS_COUNT: usize = 10000;
pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
@@ -4695,17 +4696,6 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
}
// Reusing a held writer additionally requires the same quorum, so repeated
// clients must not give different weight to an otherwise identical domain.
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
!left.is_empty()
&& left.len() == right.len()
&& left.iter().all(|client| {
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
})
}
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5655,9 +5645,9 @@ impl SetDisks {
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
// Distributed lock clients receive the raw ObjectKey; the pool
// namespace is not part of the key stored by their backends.
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
(true, true) => {
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
}
_ => false,
}
}
@@ -9892,7 +9882,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
async fn namespace_lock_domain_includes_pool_namespace() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9931,25 +9921,11 @@ mod tests {
"sets in the same pool share the object namespace lock domain"
);
assert!(
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
"different pools using identical backends lock the same raw object key"
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
"different pool namespaces must not be deduplicated solely by identical clients"
);
}
#[test]
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
assert!(!same_distributed_lock_quorum(
&[a.clone(), a.clone(), b.clone()],
&[a.clone(), b.clone(), b]
));
assert!(!same_distributed_lock_quorum(&[], &[]));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn streaming_reader_holds_read_lock_until_eof() {
@@ -141,7 +141,6 @@ pub(crate) struct DeleteRecoverySummary {
#[derive(Clone)]
pub(crate) struct DeleteRecoveryContext<'a> {
pub incarnation: Option<Uuid>,
pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
pub dry_run: bool,
pub explicit_delete: bool,
/// Only the locked heal path may repair a live marker-creation source.
@@ -157,7 +156,6 @@ impl<'a> DeleteRecoveryContext<'a> {
pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
Self {
incarnation,
work_budget: None,
dry_run: false,
explicit_delete: false,
repair_marker_source: false,
@@ -232,54 +230,19 @@ async fn inspect(
bucket: &str,
object: &str,
transaction: Uuid,
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
) -> Vec<Result<DeleteRecoveryInspection>> {
join_all(disks.iter().map(|disk| async move {
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
let state = if let Some(budget) = budget {
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
let reply = disk
.inspect_delete_transaction_bounded(bucket, object, transaction, admitted)
.await?;
if reply.visited > admitted {
return Err(DiskError::FileCorrupt);
}
budget.charge(reply.visited);
reply.inspection
} else {
disk.inspect_delete_transaction(bucket, object, transaction).await?
};
let state = disk
.as_ref()
.ok_or(DiskError::DiskNotFound)?
.inspect_delete_transaction(bucket, object, transaction)
.await?;
validate(&state, object, transaction)?;
Ok(state)
}))
.await
}
async fn discover_bounded(
disk: &DiskStore,
bucket: &str,
object: &str,
budget: &crate::object_api::DeleteRecoveryWorkBudget,
) -> Result<Vec<DeleteTransactionId>> {
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
let page = disk.discover_delete_transactions_bounded(bucket, object, admitted).await?;
if page.visited > admitted
|| page.transactions.len() > 256
|| page
.transactions
.iter()
.any(|id| id.object != object || id.transaction.is_nil())
|| (!page.complete && !page.transactions.is_empty())
{
return Err(DiskError::FileCorrupt);
}
budget.charge(page.visited);
if !page.complete {
return Err(DiskError::DeleteRecoveryWorkLimit);
}
Ok(page.transactions)
}
async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
let mut cursor = None;
let mut ids = BTreeSet::new();
@@ -310,92 +273,30 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
.collect())
}
#[derive(Clone, Copy)]
struct RecoveryWorkScope<'a> {
namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
}
async fn settle_outcome(
disk: &DiskStore,
bucket: &str,
state: &DeleteRecoveryInspection,
action: DeleteRecoveryAction,
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
scope: RecoveryWorkScope<'_>,
) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
use crate::disk::DeleteRecoverySettlementOutcome;
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let outcome = if let Some(budget) = scope.budget {
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
let reply = disk
.settle_delete_transaction_bounded(
bucket,
crate::disk::BoundedDeleteRecoverySettlement {
object: state.id.object.clone(),
transaction: state.id.transaction,
expected_fingerprint: state.fingerprint,
action,
max_entries: admitted,
},
scope.namespace_owner.cloned(),
)
.await?;
if reply.visited > admitted
|| (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
{
return Err(DiskError::FileCorrupt);
}
budget.charge(reply.visited);
reply.outcome
} else {
DeleteRecoverySettlementOutcome::Inspected(Box::new(
disk.settle_delete_transaction_with_namespace_owner(
bucket,
&state.id.object,
state.id.transaction,
state.fingerprint,
action,
scope.namespace_owner.cloned(),
)
.await?,
))
};
if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
validate(next, &state.id.object, state.id.transaction)?;
}
Ok(outcome)
}
async fn settle(
disk: &DiskStore,
bucket: &str,
state: &DeleteRecoveryInspection,
action: DeleteRecoveryAction,
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
scope: RecoveryWorkScope<'_>,
namespace_owner: Option<&Arc<dyn Send + Sync>>,
) -> Result<DeleteRecoveryInspection> {
match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
Err(DiskError::DeleteRecoveryWorkLimit)
}
}
}
fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
}
fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
match outcome {
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
Err(_) => false,
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let result = disk
.settle_delete_transaction_with_namespace_owner(
bucket,
&state.id.object,
state.id.transaction,
state.fingerprint,
action,
namespace_owner.cloned(),
)
.await;
let next = result?;
validate(&next, &state.id.object, state.id.transaction)?;
Ok(next)
}
/// Each phase fans out across independent drives, and awaits all started work
@@ -406,16 +307,23 @@ async fn settle_group(
copies: &mut [DeleteRecoveryInspection],
action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
scope: RecoveryWorkScope<'_>,
namespace_owner: Option<&Arc<dyn Send + Sync>>,
removed_versions: &mut HashSet<Uuid>,
) -> Result<bool> {
let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
let action = action(copy);
async move {
match action {
Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
.await
.map(Some),
Some(action) => settle(
disk.as_ref().ok_or(DiskError::DiskNotFound)?,
bucket,
copy,
action,
can_commit,
namespace_owner,
)
.await
.map(Some),
None => Ok(None),
}
}
@@ -531,31 +439,16 @@ impl SetDisks {
true
}
};
let mut fast_receipt_quorum = false;
if can_try_local_fast_path && can_commit() {
let fast_published = join_all(disks.iter().map(|disk| async {
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
disk.try_publish_prepared_delete_local(
bucket,
object,
transaction,
context.namespace_owner.clone(),
context.work_budget.clone(),
)
.await
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
.await
}))
.await;
fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
if fast_receipt_quorum
&& fast_published
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return Ok(());
}
let all_fast_published = fast_published.len() == self.set_drive_count
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
if all_fast_published && can_commit() {
@@ -563,7 +456,6 @@ impl SetDisks {
test_hooks::record_cleanup_attempt(bucket, object);
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
let ns_owner = context.namespace_owner.clone();
let work_budget = context.work_budget.clone();
async move {
if !can_commit() {
return Ok(false);
@@ -574,24 +466,11 @@ impl SetDisks {
let Ok(Some((journal, _, _))) = pub_res else {
return Ok(false);
};
disk.try_finish_and_finalize_prepared_delete_local(
bucket,
object,
transaction,
journal,
ns_owner,
work_budget,
)
.await
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
.await
}
}))
.await;
if finalized
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return Ok(());
}
if finalized.iter().all(|res| matches!(res, Ok(true))) {
if let Some(ticket) = context.recovery_ticket.as_ref() {
self.ctx.delete_recovery_hints().ack(ticket);
@@ -600,17 +479,7 @@ impl SetDisks {
}
}
}
let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
if states
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return if fast_receipt_quorum {
Ok(())
} else {
Err(DiskError::DeleteRecoveryWorkLimit)
};
}
let states = inspect(disks, bucket, object, transaction).await;
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
&& states.iter().all(|state| {
@@ -630,50 +499,25 @@ impl SetDisks {
if state.state == DeleteRecoveryState::LegacyAmbiguous {
return Err(DiskError::FileCorrupt);
}
settle_outcome(
settle(
disk,
bucket,
state,
DeleteRecoveryAction::PublishCommit,
can_commit,
RecoveryWorkScope {
namespace_owner: context.namespace_owner.as_ref(),
budget: context.work_budget.as_deref(),
},
context.namespace_owner.as_ref(),
)
.await
}))
.await;
if !has_receipt_quorum(&published, write_quorum) {
return Err(
if published
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
DiskError::DeleteRecoveryWorkLimit
} else {
DiskError::ErasureWriteQuorum
},
);
if published
.iter()
.filter(|state| matches!(state, Ok(state) if receipt(state.state)))
.count()
< write_quorum
{
return Err(DiskError::ErasureWriteQuorum);
}
if published.iter().any(|outcome| {
matches!(
outcome,
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
)
}) {
return Ok(());
}
let published = published
.into_iter()
.map(|outcome| match outcome {
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
Err(DiskError::DeleteRecoveryWorkLimit)
}
Err(error) => Err(error),
})
.collect::<Vec<_>>();
// Physical cleanup is retryable after the certified decision. An
// unavailable participant retains the decision for its eventual return.
let ticket = context.recovery_ticket.clone();
@@ -826,7 +670,6 @@ impl SetDisks {
}
let DeleteRecoveryContext {
incarnation,
work_budget,
dry_run,
explicit_delete,
repair_marker_source,
@@ -838,13 +681,7 @@ impl SetDisks {
} = context;
let disks = self.disk_inventory().await;
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
let pages = join_all(disks.iter().flatten().map(|disk| async {
match work_budget.as_deref() {
Some(budget) => discover_bounded(disk, bucket, object, budget).await,
None => discover(disk, bucket, Some(object)).await,
}
}))
.await;
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
let mut transactions = BTreeSet::new();
let mut inspected = 0;
let mut offline = !complete_inventory;
@@ -875,7 +712,6 @@ impl SetDisks {
..Default::default()
};
let mut all_states = BTreeMap::new();
let mut registered_evidence_hint = false;
for transaction in transactions {
let mut complete = Vec::with_capacity(disks.len());
let inspections = if let Some((published_tx, published_states)) = published_hint
@@ -887,14 +723,8 @@ impl SetDisks {
{
published_states.to_vec()
} else {
inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
inspect(&disks, bucket, object, transaction).await
};
if inspections
.iter()
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
{
return Err(DiskError::DeleteRecoveryWorkLimit);
}
for state in inspections {
match state {
Ok(state) => {
@@ -906,33 +736,6 @@ impl SetDisks {
summary.blocked += 1;
return Ok(summary);
}
// Candidate UUID directories are not evidence. Only a bound, inspected
// journal can create a foreground hint; empty/partial scans rely on the
// existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
if work_budget.is_some()
&& !registered_evidence_hint
&& journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
&& journal.volume == bucket
&& journal.object == object
&& journal.transaction == transaction
&& matches!(
state.state,
DeleteRecoveryState::NewPrepared
| DeleteRecoveryState::QuorumCommitted
| DeleteRecoveryState::CommitFinished
)
{
self.ctx.delete_recovery_hints().register(
crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: self.pool_index,
set: self.set_index,
bucket: bucket.to_owned(),
object: object.to_owned(),
incarnation,
},
);
registered_evidence_hint = true;
}
}
complete.push(state);
}
@@ -964,7 +767,7 @@ impl SetDisks {
};
if let Some((transaction, source)) = prerequisite {
Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
let refreshed = inspect(&disks, bucket, object, transaction)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
@@ -1050,7 +853,7 @@ impl SetDisks {
if mutated {
// A sibling may have restored metadata or reclaimed shared
// evidence. Refresh fingerprints before acting on this group.
*copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
*copies = inspect(&disks, bucket, object, transaction)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
@@ -1073,10 +876,7 @@ impl SetDisks {
(copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
},
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?
@@ -1187,10 +987,7 @@ impl SetDisks {
)
},
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?;
@@ -1204,10 +1001,7 @@ impl SetDisks {
.then_some(DeleteRecoveryAction::FinishCommit)
},
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?;
@@ -1254,10 +1048,7 @@ impl SetDisks {
copies,
|copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?
@@ -1285,7 +1076,7 @@ impl SetDisks {
settled.push(copies);
} else {
for transaction in all_states.keys() {
let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
let fresh = inspect(&disks, bucket, object, *transaction).await;
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
summary.pending += 1;
return Ok(summary);
@@ -1295,7 +1086,7 @@ impl SetDisks {
}
for (index, mut copies) in settled.into_iter().enumerate() {
if index > 0 {
copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
copies = inspect(&disks, bucket, object, copies[0].id.transaction)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
@@ -1315,10 +1106,7 @@ impl SetDisks {
&mut copies,
|copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
can_commit,
RecoveryWorkScope {
namespace_owner: namespace_owner.as_ref(),
budget: work_budget.as_deref(),
},
namespace_owner.as_ref(),
&mut summary.removed_versions,
)
.await?
@@ -1570,459 +1358,6 @@ mod tests {
body
}
#[tokio::test]
async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
let bucket = "recover-bounded-mixed-quorum";
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &vec![73; 4096], false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let mut outcomes = Vec::new();
for (index, disk) in disks.iter().enumerate() {
let state = disk
.inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
.await
.expect("preparation inspection");
let max_entries = match index {
0 => 65536,
1 | 2 => state.visited,
_ => 1,
};
outcomes.push(
disk.settle_delete_transaction_bounded(
bucket,
crate::disk::BoundedDeleteRecoverySettlement {
object: "object".to_owned(),
transaction,
expected_fingerprint: state.inspection.fingerprint,
action: DeleteRecoveryAction::PublishCommit,
max_entries,
},
None,
)
.await
.map(|reply| reply.outcome),
);
}
assert!(
matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
);
assert!(matches!(
&outcomes[1],
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
));
assert!(outcomes[3].is_err());
assert!(has_receipt_quorum(&outcomes, 3));
assert!(!has_receipt_quorum(&outcomes, 4));
outcomes[2] = Err(DiskError::FileAccessDenied);
assert!(
!has_receipt_quorum(&outcomes, 3),
"a write failure cannot substitute for a durable receipt"
);
}
#[tokio::test]
async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
use crate::object_api::DeleteRecoveryWorkBudget;
use crate::storage_api_contracts::object::ObjectToDelete;
let bucket = "recover-bounded-request";
let (dirs, store, _) = fixture(bucket).await;
for dir in &dirs {
let prefix = dir.path().join(bucket).join("prefix");
std::fs::create_dir_all(&prefix).expect("physical prefix");
for _ in 0..64 {
std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("legitimate child");
}
}
let error = store
.delete_object(
bucket,
"prefix",
ObjectOptions {
delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
..Default::default()
},
)
.await
.expect_err("incomplete discovery is not absence");
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
for incarnation in [
None,
Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation"),
),
] {
assert!(
!store
.ctx
.delete_recovery_hints()
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "prefix".to_owned(),
incarnation,
}),
"an unproven prefix must not move its scan into background recovery"
);
}
for distinct_versions in [false, true] {
let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
let objects = (0..1000)
.map(|_| ObjectToDelete {
object_name: "prefix".to_owned(),
version_id: distinct_versions.then(Uuid::new_v4),
..Default::default()
})
.collect();
let (_, errors) = store
.delete_objects(
bucket,
objects,
ObjectOptions {
delete_recovery_work_budget: Some(Arc::clone(&budget)),
..Default::default()
},
)
.await;
assert!(
errors
.iter()
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
"repeated actions must not each obtain a new allowance: {distinct_versions}"
);
assert!(dirs.iter().all(|dir| {
std::fs::read_dir(dir.path().join(bucket).join("prefix"))
.map(|entries| entries.count() == 64)
.unwrap_or(false)
}));
}
let error = store
.delete_object(bucket, "prefix", ObjectOptions::default())
.await
.expect_err("ordinary missing key");
assert!(
crate::error::is_err_object_not_found(&error),
"small complete inventory retains no-op semantics: {error:?}"
);
}
#[tokio::test]
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
use crate::layout::endpoint::Endpoint;
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::runtime::instance::InstanceContext;
let mut dirs = Vec::new();
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for set_index in 0..2 {
for disk_index in 0..4 {
let dir = tempfile::tempdir().expect("temporary disk");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(set_index);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
dirs.push(dir);
}
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 2,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("bounded-recovery-{pool_index}"),
platform: "test".to_owned(),
});
}
let endpoints = EndpointServerPools(pools);
let ctx = Arc::new(InstanceContext::new());
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
.await
.expect("local disks");
let cancel = tokio_util::sync::CancellationToken::new();
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
.await
.expect("two pools with two sets");
cancel.cancel();
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "recover-bounded-fanout";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
let mut keys = BTreeSet::new();
for pool in &store.pools {
for set in &pool.disk_set {
keys.insert(
candidates
.iter()
.find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
.expect("fixture key for set")
.clone(),
);
}
}
let keys = keys.into_iter().collect::<Vec<_>>();
for pool in &store.pools {
for set in &pool.disk_set {
assert!(
keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
"fixture must cover every set"
);
}
}
for dir in &dirs {
for key in &keys {
let prefix = dir.path().join(bucket).join(key);
std::fs::create_dir_all(&prefix).expect("physical prefix");
for _ in 0..8 {
std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("child");
}
}
}
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
let objects = (0..1000)
.map(|index| crate::storage_api_contracts::object::ObjectToDelete {
object_name: keys[index % keys.len()].clone(),
version_id: Some(Uuid::new_v4()),
..Default::default()
})
.collect();
let (_, errors) = store
.delete_objects(
bucket,
objects,
ObjectOptions {
delete_recovery_work_budget: Some(Arc::clone(&budget)),
..Default::default()
},
)
.await;
assert!(
errors
.iter()
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
"all pools and sets must share the finite allowance"
);
assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
assert!(dirs.iter().all(|dir| {
std::fs::read_dir(dir.path().join(bucket).join(&keys[0]))
.map(|entries| entries.count() == 8)
.unwrap_or(false)
}));
}
#[tokio::test]
async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
let bucket = "recover-bounded-fast-commit";
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
// Inline cleanup needs no directory enumeration and may legitimately
// finish with zero scan credits. Require real part files here.
for disk in &disks {
let info = disk
.read_version("", bucket, "object", "", &ReadOptions::default())
.await
.expect("read non-inline source");
assert!(!info.inline_data(), "the fixture must require physical part cleanup");
let data_dir = info.data_dir.expect("physical data directory");
let part = std::path::PathBuf::from(disk.endpoint().to_string())
.join(bucket)
.join("object")
.join(data_dir.to_string())
.join("part.1");
assert!(part.is_file(), "the cleanup scan must encounter a real part");
}
store
.delete_object(
bucket,
"object",
ObjectOptions {
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
..Default::default()
},
)
.await
.expect("a cached prepared receipt commits independently of cleanup budget");
assert!(
store
.ctx
.delete_recovery_hints()
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "object".to_owned(),
incarnation: Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation")
),
})
);
assert!(
!disks[0]
.discover_delete_transactions(bucket, Some("object"), None, 128)
.await
.expect("retained recovery proof")
.transactions
.is_empty()
);
store
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
.await
.expect("background finishes committed cleanup");
}
#[tokio::test]
async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
let bucket = "recover-bounded-candidate";
let (dirs, store, _) = fixture(bucket).await;
let transaction = Uuid::new_v4();
for directory in &dirs {
let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
std::fs::create_dir_all(&candidate).expect("candidate path");
std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
}
assert!(
store
.delete_object(bucket, "candidate", ObjectOptions::default())
.await
.is_err()
);
for incarnation in [
None,
Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation"),
),
] {
assert!(
!store
.ctx
.delete_recovery_hints()
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "candidate".to_owned(),
incarnation,
})
);
}
}
#[tokio::test]
async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
let bucket = "recover-bounded-evidence";
let (_dirs, store, disks) = fixture(bucket).await;
let payload = vec![71; 4096];
put(&store, bucket, &payload, false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let before = disks[0]
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("before refusal");
let error = store
.delete_object(
bucket,
"object",
ObjectOptions {
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
..Default::default()
},
)
.await
.expect_err("no incomplete recovery admission");
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
let after = disks[0]
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("after refusal");
assert_eq!(before.fingerprint, after.fingerprint);
store
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
.await
.expect("background-compatible recovery");
assert_eq!(body(&store, bucket, None).await, payload);
}
#[tokio::test]
async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
let bucket = "recover-bounded-commit";
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &vec![72; 4096], false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let set = &store.pools[0].disk_set[0];
let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
pool: 0,
set: 0,
bucket: bucket.to_owned(),
object: "object".to_owned(),
incarnation: None,
};
let ticket = set.ctx.delete_recovery_hints().register(key.clone());
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
// This hook disables local fast cleanup and runs only after receipt
// publication and the shared recovery path's key-wide GC inspection.
// Exhausting preparation instead would correctly reject the delete.
test_hooks::before_gc_finalize(bucket, "object", {
let budget = Arc::clone(&budget);
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
Arc::new(move || {
let budget = Arc::clone(&budget);
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
Box::pin(async move {
assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
})
})
});
set.publish_delete_commit(
bucket,
"object",
transaction,
&set.disk_inventory().await,
set.set_drive_count / 2 + 1,
DeleteRecoveryContext {
work_budget: Some(Arc::clone(&budget)),
recovery_ticket: ticket,
..DeleteRecoveryContext::new(None, &|| true)
},
)
.await
.expect("the durable decision still succeeds");
assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
assert_eq!(budget.remaining(), 0);
assert!(set.ctx.delete_recovery_hints().is_pending(&key));
let state = disks[0]
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("retained decision");
assert!(receipt(state.state));
store
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
.await
.expect("eventual cleanup");
assert!(
disks[0]
.discover_delete_transactions(bucket, Some("object"), None, 128)
.await
.expect("cleaned proofs")
.transactions
.is_empty()
);
}
#[tokio::test]
async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
let bucket = "recover-undecided";
+6 -89
View File
@@ -95,8 +95,13 @@ impl SetDisks {
bucket: &str,
object: &str,
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
let resource = ObjectKey::with_version(
bucket,
object_name_disk_identity(object).as_ref(),
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
);
let guard = self
.namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
.namespace_lock_for_key(resource)
.await
.get_write_lock(super::super::get_lock_acquire_timeout())
.await
@@ -106,94 +111,6 @@ impl SetDisks {
Ok(Arc::new(guard))
}
fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
ObjectKey::with_version(
bucket,
object_name_disk_identity(object).as_ref(),
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
)
}
/// Batched write-lock acquisition for the whole key set of one bulk
/// mutation: one lock round instead of one quorum round-trip per key.
///
/// Bulk delete is the only multi-key write-lock holder. The batched round
/// is deadlock-free by construction (a single `lock_batch` RPC per lock
/// client, and every lock server orders concurrent batches by the same
/// global (shard, key) order), so unlike an optimistic probe wave it
/// needs no canonical serial fallback. Outcomes are per key and use the
/// same error shape as the single-key paths.
async fn namespace_write_locks_batch(
&self,
bucket: &str,
resources: &[ObjectKey],
timeout: std::time::Duration,
mode: &'static str,
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
if resources.is_empty() {
return Vec::new();
}
let lock = self.namespace_lock_for_key(resources[0].clone()).await;
lock.get_write_locks_batch(resources, timeout)
.await
.into_iter()
.zip(resources)
.map(|(result, resource)| {
result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
})
.collect()
}
/// Batched raw namespace write locks for a bulk delete's key set. The
/// resource keys use the same disk-identity folding as `new_ns_lock`, so
/// aliases (`a\b` vs `a/b`) still share one lock.
pub(crate) async fn acquire_object_write_locks_batch(
&self,
bucket: &str,
objects: &[&str],
timeout: std::time::Duration,
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
let resources: Vec<ObjectKey> = objects
.iter()
.map(|object| ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object_name_disk_identity(object).as_ref()),
version: None,
})
.collect();
self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
}
/// Batched delete-transaction locks for a bulk delete's key set; see
/// [`Self::acquire_delete_transaction_lock`] for the resource shape.
pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
&self,
bucket: &str,
objects: &[&str],
timeout: std::time::Duration,
) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
let resources: Vec<ObjectKey> = objects
.iter()
.map(|object| self.delete_transaction_resource(bucket, object))
.collect();
let mut guards = Vec::with_capacity(resources.len());
for (object, result) in objects.iter().zip(
self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
.await,
) {
match result {
Ok(guard) => {
#[cfg(test)]
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
let _ = object;
guards.push(Ok(Arc::new(guard)));
}
Err(error) => guards.push(Err(error)),
}
}
guards
}
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
match err {
File diff suppressed because it is too large Load Diff
+45 -627
View File
@@ -3914,7 +3914,6 @@ impl SetDisks {
bucket,
object,
DeleteRecoveryContext {
work_budget: opts.delete_recovery_work_budget.clone(),
explicit_delete: true,
transaction_guard: transaction_guard.cloned(),
namespace_owner: Some(Arc::new((
@@ -4790,27 +4789,6 @@ impl SetDisks {
.await?;
}
// Lock order: lifecycle, metadata transaction, then object namespace.
// No body reads occur below this point. The snapshot must describe the
// configuration at commit, not the pre-body advisory object lookup.
let mut commit_opts;
let opts = if opts.defer_put_object_lock_snapshot {
commit_opts = opts.clone();
let expected = source_bucket_incarnation_id
.ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
let store = metadata_sys::object_store_in(&self.ctx).await?;
let snapshot = store.object_lock_config_snapshot_for_single_put(bucket).await?;
if !snapshot.is_valid_for_destructive_put(store.id, bucket, expected) {
return Err(Error::other("PUT bucket generation or configuration changed while staging"));
}
snapshot.add_lock_fences(&mut commit_opts);
commit_opts.object_lock_config_snapshot = Some(snapshot);
commit_opts.defer_put_object_lock_snapshot = false;
&commit_opts
} else {
opts
};
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
#[cfg(test)]
{
@@ -4980,20 +4958,7 @@ impl SetDisks {
);
}
// A write without a version id to a key that is not versioning-Enabled
// replaces the key's null version in place, so the WORM gate must judge
// that version too. The request-layer pre-check reads the latest
// version, which is not the one replaced when a UUID version or a
// delete marker sits above a retained null version.
let worm_gate_version_id = opts.version_id.clone().or_else(|| {
(!opts.versioned
&& opts
.object_lock_config_snapshot
.as_ref()
.is_some_and(|snapshot| !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)))
.then(|| Uuid::nil().to_string())
});
if let Some(version_id) = worm_gate_version_id.as_deref()
if let Some(version_id) = opts.version_id.as_deref()
&& !is_meta_bucketname(bucket)
{
let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer();
@@ -5013,10 +4978,9 @@ impl SetDisks {
.await;
match current {
Ok(existing) => {
let object_lock_config = opts
.object_lock_config_snapshot
.as_deref()
.ok_or_else(|| Error::other("destructive PUT is missing its Object Lock configuration snapshot"))?;
let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
})?;
// The WORM gate protects the locked version from local
// overwrites; an authorized replication write passes it
// only when the LWW merge below will judge every
@@ -5047,8 +5011,6 @@ impl SetDisks {
}
}
}
// A null delete marker has no protected payload to replace.
Err(StorageError::MethodNotAllowed) if version_id == Uuid::nil().to_string() => {}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
}
@@ -5304,7 +5266,6 @@ impl SetDisks {
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
let commit_publication_guard = publication_commit_guard.take();
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
let commit_object_lock_config_snapshot = opts.object_lock_config_snapshot.clone();
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
// A scanner publication scope owns the movement permit until the
@@ -5367,7 +5328,6 @@ impl SetDisks {
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
let mut _publication_guard = commit_publication_guard;
let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
let mut _object_lock_config_snapshot = commit_object_lock_config_snapshot;
let mut _decommission_capacity_guard = commit_decommission_capacity_guard;
let mut quota_reservation = quota_reservation;
let rename_stage_start = Instant::now();
@@ -5701,7 +5661,6 @@ impl SetDisks {
let object_lock_guard = _object_lock_guard.take();
let publication_guard = _publication_guard.take();
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
let object_lock_config_snapshot = _object_lock_config_snapshot.take();
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
let decommission_capacity_guard = _decommission_capacity_guard.take();
let cleanup_bucket = commit_bucket.clone();
@@ -5724,7 +5683,6 @@ impl SetDisks {
write_context,
publication_guard,
bucket_lifecycle_guard,
object_lock_config_snapshot,
decommission_object_lock_guard,
decommission_capacity_guard,
),
@@ -5746,7 +5704,6 @@ impl SetDisks {
write_context,
publication_guard,
bucket_lifecycle_guard,
object_lock_config_snapshot,
decommission_object_lock_guard,
decommission_capacity_guard,
),
@@ -5760,7 +5717,6 @@ impl SetDisks {
drop(write_context);
drop(publication_guard);
drop(bucket_lifecycle_guard);
drop(object_lock_config_snapshot);
cleanup_runtime::run(cleanup_reservation, async move {
let zero_target_cleanup = targets.is_empty();
rustfs_io_metrics::record_put_object_stage_duration(
@@ -9175,9 +9131,6 @@ impl SetDisks {
&disks,
write_quorum,
DeleteRecoveryContext {
work_budget: commit_context
.options
.and_then(|options| options.delete_recovery_work_budget.clone()),
recovery_ticket,
transaction_guard: commit_context.transaction_guard.clone(),
namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
@@ -9351,60 +9304,6 @@ impl SetDisks {
}
impl SetDisks {
/// Judge the version a versionless destructive delete removes.
///
/// For a key that is not versioning-`Enabled`, a delete without a version
/// id removes the `null` version, while the preceding Object Lock check
/// judged the key's latest entry. When a UUID version or delete marker
/// sits above a retained `null` version, the two differ and the retained
/// version must be judged on its own. The caller holds the object's write
/// lock. Plain unversioned keys and versioning-`Enabled` keys keep a single
/// lookup: their latest entry is the version removed.
///
/// The lookup future is boxed from a synchronous wrapper so it leaves the
/// delete futures' frames: `delete_object` already nests the boxed batch
/// path, and inlining a second `get_object_info` overflowed the 4 MiB test
/// stack on Linux.
fn check_null_version_object_lock_delete<'a>(
&'a self,
bucket: &'a str,
object: &'a str,
latest: &'a ObjectInfo,
opts: &'a ObjectOptions,
) -> impl Future<Output = Result<()>> + Send + 'a {
Box::pin(self.check_null_version_object_lock_delete_inner(bucket, object, latest, opts))
}
async fn check_null_version_object_lock_delete_inner(
&self,
bucket: &str,
object: &str,
latest: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<()> {
if is_meta_bucketname(bucket)
|| opts.version_id.is_some()
|| set_disk_delete_creates_delete_marker(opts)
|| opts.transition.expire_restored
|| latest.version_id.is_none_or(|version| version.is_nil())
{
return Ok(());
}
let null_opts = ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
no_lock: true,
metadata_cache_safe: false,
..opts.clone()
};
match self.get_object_info(bucket, object, &null_opts).await {
Ok(existing) => check_object_lock_delete(&self.ctx, bucket, object, &existing, &null_opts).await,
// A null delete marker has no protected payload to remove.
Err(StorageError::MethodNotAllowed) => Ok(()),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
Err(err) => Err(err),
}
}
/// Read current metadata without cache admission or read-side repair. The
/// outer batch owner already holds the sorted namespace write-lock set.
pub(crate) async fn conditional_delete_current_snapshot(
@@ -9439,76 +9338,6 @@ where
futures::future::try_join_all(probes).await.map(|_| ()).map_err(Error::from)
}
/// Per-key source-read inputs for the bulk-delete admission loop. The bounded
/// metadata pre-fetch and the loop itself must resolve these identically, so
/// both sides derive them here.
struct BatchDeleteSourceInputs {
replication_object_name: String,
explicit_null_version: bool,
version_id: Option<Uuid>,
versioned: bool,
version_suspended: bool,
check_opts: ObjectOptions,
object_lock_check_required: bool,
replicate_delete: bool,
needs_source_read: bool,
}
fn batch_delete_source_inputs(
bucket: &str,
dobj: &ObjectToDelete,
opts: &ObjectOptions,
delete_config_snapshot: &DeleteReplicationConfigSnapshot,
object_lock_config: &Arc<ObjectLockConfigSnapshot>,
guarded: bool,
) -> BatchDeleteSourceInputs {
let replication_object_name = decode_dir_object(&dobj.object_name);
let explicit_null_version = is_explicit_null_version(dobj.version_id);
let version_id = delete_file_info_version_id(dobj.version_id);
let (versioned, version_suspended) = delete_config_snapshot
.versioning_config()
.delete_state(replication_object_name.as_str());
let check_opts = ObjectOptions {
version_id: dobj.version_id.map(|version_id| version_id.to_string()),
versioned,
version_suspended,
object_lock_delete: opts.object_lock_delete.clone(),
object_lock_config_snapshot: Some(Arc::clone(object_lock_config)),
no_lock: true,
suppress_read_repair: guarded,
..Default::default()
};
// A missing bucket Object Lock configuration does not prove that
// persisted object metadata is clean: migrated or corrupt data may
// still carry an explicit retention or legal hold. Only a real
// delete-marker creation can skip the object-level WORM check.
let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
let replication_needs_source = replicate_delete
&& (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
// Guarded explicit-version deletes are rejected before any read and must
// not join the pre-fetch wave.
let admission_error = guarded && dobj.version_id.is_some();
let needs_source_read = !admission_error
&& (object_lock_check_required
|| replication_needs_source
|| opts.tier_delete_journal_api.is_some()
|| dobj.expected_identity.is_some()
|| dobj.version_id.is_some());
BatchDeleteSourceInputs {
replication_object_name,
explicit_null_version,
version_id,
versioned,
version_suspended,
check_opts,
object_lock_check_required,
replicate_delete,
needs_source_read,
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
type Error = Error;
@@ -9741,26 +9570,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if let Some(etag) = &src_info.etag {
replacement_metadata.insert("etag".to_owned(), etag.clone());
}
// ObjectInfo externalizes these fields; copy callers supply the effective
// values, including empty tags/absent expiry for explicit replacement.
if !src_info.user_tags.is_empty() {
replacement_metadata
.entry(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string())
.or_insert_with(|| (*src_info.user_tags).clone());
}
if let Some(expires) = src_info.expires
&& !replacement_metadata.contains_key("expires")
{
replacement_metadata.insert(
"expires".to_string(),
expires
.format(&time::format_description::well_known::Rfc3339)
.map_err(Error::other)?,
);
}
fi.metadata = replacement_metadata.clone();
let mod_time = dst_opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc);
let mod_time = OffsetDateTime::now_utc();
fi.mod_time = Some(mod_time);
fi.version_id = version_id;
fi.versioned = src_opts.versioned || src_opts.version_suspended;
@@ -9883,7 +9695,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
objects: Vec<ObjectToDelete>,
mut opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
opts.ensure_delete_recovery_work_budget();
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut accounting = vec![None; objects.len()];
let delete_config_snapshot = opts
@@ -9925,7 +9736,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let unique_lock_count = batch.requests.len();
let mut failed_map = HashMap::new();
let mut local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
let mut locked_objects = HashSet::new();
// Instance-scoped, not the ambient facade (backlog#1052) — see
@@ -9942,7 +9753,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
(failed_map, locked_objects, dist_batch_lock_ids) = self.acquire_dist_delete_object_locks_batch(&batch).await;
} else {
let batch_result = self.local_lock_manager.acquire_locks_batch(batch).await;
local_batch_guards = batch_result.guards;
_local_batch_guards = batch_result.guards;
for key in batch_result.successful_locks {
locked_objects.insert(key.object.as_ref().to_string());
@@ -9984,17 +9795,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// Every raw batch lock is acquired first. Canonical sorting also
// gives aliases of one disk key exactly one transaction owner.
let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
// Bulk delete is the only multi-key transaction-lock holder; one
// batched lock round replaces one quorum round-trip per key and is
// deadlock-free by construction (a single `lock_batch` RPC per lock
// client, global (shard, key) order at each server). Per-key
// failures keep the set layer's per-key error reporting.
let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
let acquired = self
.acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
.await;
for (key, result) in keys.into_iter().zip(acquired) {
match result {
for key in keys {
match self.acquire_delete_transaction_lock(bucket, &key).await {
Ok(guard) => {
opts.add_namespace_lock_guard(&guard);
transaction_guards.insert(key, guard);
@@ -10030,54 +9832,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
},
}
};
// Per-key read inputs are resolved exactly once; the pre-fetch and the
// admission loop below consume this table so the two cannot disagree
// on eligibility or read options.
let mut source_inputs: Vec<Option<BatchDeleteSourceInputs>> = Vec::with_capacity(objects.len());
for dobj in &objects {
let guarded = current_generations.is_some_and(|generations| generations.contains_key(&dobj.object_name));
source_inputs.push(Some(batch_delete_source_inputs(
bucket,
dobj,
&opts,
&delete_config_snapshot,
&object_lock_config,
guarded,
)));
}
// Bounded-parallel pre-fetch of the source metadata each admitted key
// needs below. These reads are pure quorum lookups under the locks
// already held; sequencing one metadata fan-out per key made a
// 1000-key batch pay a full round of drive reads at a time.
const DELETE_OBJECTS_SOURCE_READ_CONCURRENCY: usize = 16;
let mut source_reads: HashMap<usize, (ObjectInfo, FileInfo, Option<Error>)> = HashMap::with_capacity(objects.len());
{
let mut pending = Vec::new();
for (i, source) in source_inputs.iter().enumerate() {
if del_errs[i].is_some() {
continue;
}
let Some(source) = source else {
continue;
};
if !source.needs_source_read {
continue;
}
let object_name = objects[i].object_name.clone();
let check_opts = &source.check_opts;
pending.push(async move {
let (goi, file_info, _write_quorum, gerr) = self
.get_object_info_fileinfo_and_quorum(bucket, &object_name, check_opts)
.await;
(i, (goi, file_info, gerr))
});
}
let reads = futures::stream::iter(pending)
.buffer_unordered(DELETE_OBJECTS_SOURCE_READ_CONCURRENCY)
.collect::<Vec<_>>()
.await;
source_reads.extend(reads);
}
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
let mut tier_reference_leases: Vec<(usize, String, Option<TierDestinationId>)> = Vec::new();
let mut tier_free_version_receipt_candidates: HashMap<usize, TierFreeVersionReceiptCandidate> = HashMap::new();
@@ -10099,30 +9853,41 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
));
continue;
}
let Some(source) = source_inputs[i].take() else {
continue;
};
let BatchDeleteSourceInputs {
replication_object_name,
explicit_null_version,
version_id,
let replication_object_name = decode_dir_object(&dobj.object_name);
let explicit_null_version = is_explicit_null_version(dobj.version_id);
let version_id = delete_file_info_version_id(dobj.version_id);
let (versioned, version_suspended) = delete_config_snapshot
.versioning_config()
.delete_state(replication_object_name.as_str());
let check_opts = ObjectOptions {
version_id: dobj.version_id.map(|version_id| version_id.to_string()),
versioned,
version_suspended,
check_opts,
object_lock_check_required,
replicate_delete,
needs_source_read,
} = source;
let (mut goi, mut authoritative_file_info, mut gerr) = if needs_source_read {
match source_reads.remove(&i) {
Some(read) => read,
None => {
let (goi, file_info, _write_quorum, gerr) = self
.get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
.await;
(goi, file_info, gerr)
}
}
object_lock_delete: opts.object_lock_delete.clone(),
object_lock_config_snapshot: Some(Arc::clone(&object_lock_config)),
no_lock: true,
suppress_read_repair: guarded,
..Default::default()
};
// A missing bucket Object Lock configuration does not prove that
// persisted object metadata is clean: migrated or corrupt data may
// still carry an explicit retention or legal hold. Only a real
// delete-marker creation can skip the object-level WORM check.
let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
let replication_needs_source = replicate_delete
&& (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required
|| replication_needs_source
|| opts.tier_delete_journal_api.is_some()
|| dobj.expected_identity.is_some()
|| dobj.version_id.is_some()
{
let (goi, file_info, _write_quorum, gerr) = self
.get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
.await;
(goi, file_info, gerr)
} else {
(ObjectInfo::default(), FileInfo::default(), None)
};
@@ -10139,23 +9904,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&& !guarded
&& goi.name.is_empty()
{
// Recovery can restore or retire versions of this key
// (rolling back an interrupted prepared delete brings a
// retained version back). A pre-fetched read for a later
// duplicate of the same name would still report the stale
// miss and skip its object-lock check — bypassing retention —
// so evict those entries and let the loop re-read them live.
for (other_index, other) in objects.iter().enumerate() {
if other_index != i && other.object_name == dobj.object_name {
source_reads.remove(&other_index);
}
}
let recovery = self
.recover_delete_key(
bucket,
&dobj.object_name,
DeleteRecoveryContext {
work_budget: opts.delete_recovery_work_budget.clone(),
dry_run: !opts.no_lock,
transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
namespace_owner: opts.delete_mutation_owner.clone(),
@@ -10269,17 +10022,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
del_errs[i] = Some(err);
continue;
}
// Not gated on `source_missing`: with a delete marker on top the
// latest-version lookup reports the key as missing while the null
// version below it may still be retained.
if object_lock_check_required
&& let Err(err) = self
.check_null_version_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts)
.await
{
del_errs[i] = Some(err);
continue;
}
if opts.tier_delete_journal_api.is_some()
&& !source_missing
@@ -10735,10 +10477,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// response can hide a persisted decision and must never roll back.
let groups = groups.collect::<Vec<_>>();
published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
// Journal publish fans out per object per drive; keep enough
// publications in flight to feed the drives' durable-mutation
// lanes (8 workers each) across a 16-drive set.
const DELETE_JOURNAL_PUBLISH_CONCURRENCY: usize = 32;
let publications = futures::stream::iter(groups.into_iter().map(|index: usize| {
let group = &vers[index];
let recovery_ticket = recovery_tickets[index].clone();
@@ -10764,7 +10502,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
disks,
self.set_drive_count / 2 + 1,
DeleteRecoveryContext {
work_budget: opts.delete_recovery_work_budget.clone(),
recovery_ticket,
namespace_owner,
transaction_guard,
@@ -10775,7 +10512,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
(index, result)
}
}))
.buffer_unordered(DELETE_JOURNAL_PUBLISH_CONCURRENCY)
.buffer_unordered(16)
.collect::<Vec<_>>()
.await;
for (index, result) in publications {
@@ -10915,14 +10652,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// Deferred recovery takes fresh raw and transaction locks. Discard the
// completed leases' deadlines while retaining every caller-owned fence.
opts.namespace_lock_fence = outer_namespace_lock_fence;
// Release inner owners before reacquiring any raw key lock. The raw
// batch guards must go too: deferred recovery takes a fresh write lock
// on the same keys below, and a still-held local batch guard would
// deadlock it against itself (the distributed path releases its batch
// ids in the same region).
// Release inner owners before reacquiring any raw key lock.
drop(namespace_owner);
drop(transaction_guards);
drop(local_batch_guards);
// TODO(backlog): support partial object deletion for multi-part objects
@@ -10948,7 +10680,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
bucket,
&object.object_name,
DeleteRecoveryContext {
work_budget: opts.delete_recovery_work_budget.clone(),
explicit_delete,
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
@@ -11007,7 +10738,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self, opts))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
opts.ensure_delete_recovery_work_budget();
let _scope_outcome_guard = opts
.scanner_publication_commit_scope
.clone()
@@ -11304,7 +11034,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
bucket,
object,
DeleteRecoveryContext {
work_budget: opts.delete_recovery_work_budget.clone(),
transaction_guard: transaction_guard.clone(),
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
@@ -11365,8 +11094,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
opts.precondition_check(&goi)?;
check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?;
}
self.check_null_version_object_lock_delete(bucket, object, &goi, &opts)
.await?;
let delete_marker_purge_candidate = opts
.version_id
.as_deref()
@@ -11916,10 +11643,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// as a second object header set: forwarding SSE intent or wrapped DEKs
// would request a second encryption pass and disclose local envelope
// material to the remote provider.
transition_meta.retain(|key, _| {
!rustfs_utils::http::is_replication_stripped_encryption_key(key)
&& !rustfs_utils::http::is_snowball_transport_key(key)
});
transition_meta.retain(|key, _| !rustfs_utils::http::is_replication_stripped_encryption_key(key));
transition_meta.insert("name".to_string(), object.to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
@@ -16395,170 +16119,6 @@ mod metadata_mutation_generation_tests {
assert_retired(&set_disks, &metadata_key).await;
}
#[tokio::test]
async fn restore_metadata_copy_preserves_tags_expiry_and_version_order() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use s3s::dto::RestoreRequest;
use time::format_description::well_known::Rfc3339;
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "restore-metadata-preservation";
for disk in &disks {
disk.make_volume(bucket).await.expect("create bucket");
}
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
let latest_time = old_time + time::Duration::days(1);
let expires = latest_time + time::Duration::days(365);
for deleted in [false, true] {
let object = if deleted { "deleted" } else { "live" };
let tags = "classification=secret&owner=alice";
let old = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(b"old protected content".to_vec()),
&ObjectOptions {
versioned: true,
mod_time: Some(old_time),
write_completion: crate::object_api::WriteCompletion::TailDrained,
user_defined: HashMap::from([
(AMZ_OBJECT_TAGGING.to_string(), tags.to_string()),
("x-amz-tagging".to_string(), tags.to_string()),
("expires".to_string(), expires.format(&Rfc3339).expect("expiry")),
]),
..Default::default()
},
)
.await
.expect("write historical version");
let latest = if deleted {
let marker = FileInfo {
version_id: Some(Uuid::new_v4()),
mod_time: Some(latest_time),
deleted: true,
versioned: true,
..Default::default()
};
for disk in &disks {
disk.write_metadata(bucket, bucket, object, marker.clone())
.await
.expect("publish current delete marker");
}
marker.version_id
} else {
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(b"latest content".to_vec()),
&ObjectOptions {
versioned: true,
mod_time: Some(latest_time),
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write latest version")
.version_id
};
let opts = ObjectOptions {
version_id: old.version_id.map(|id| id.to_string()),
versioned: true,
mod_time: Some(old_time),
..Default::default()
};
let mut source = set.get_object_info(bucket, object, &opts).await.expect("read old metadata");
assert_eq!(source.user_tags.as_str(), tags);
assert_eq!(source.expires, Some(expires));
assert!(!source.user_defined.contains_key(AMZ_OBJECT_TAGGING));
assert!(source.user_defined.contains_key("x-amz-tagging"));
source.metadata_only = true;
Arc::make_mut(&mut source.user_defined)
.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
set.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
.await
.expect("restore metadata update");
let restored = set
.get_object_info(bucket, object, &opts)
.await
.expect("read restored metadata");
assert_eq!(restored.user_tags.as_str(), tags);
assert_eq!(restored.expires, Some(expires));
assert_eq!(restored.mod_time, Some(old_time));
for disk in &disks {
let bytes = disk
.read_all(bucket, &format!("{object}/xl.meta"))
.await
.expect("persisted metadata");
let meta = FileMeta::load(&bytes).expect("decode versions");
let versions = meta.list_versions(bucket, object, false).expect("list versions");
assert_eq!(versions.len(), 2);
assert_eq!(versions[0].version_id, latest);
assert_eq!(versions[0].mod_time, Some(latest_time));
assert_eq!(versions[0].deleted, deleted);
assert_eq!(versions[1].version_id, old.version_id);
assert_eq!(versions[1].mod_time, Some(old_time));
}
let restore_opts = put_restore_opts(bucket, object, &RestoreRequest::default(), &restored)
.await
.expect("background restore options");
assert_eq!(restore_opts.user_defined.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some(tags));
assert_eq!(restore_opts.user_defined.get("expires"), Some(&expires.format(&Rfc3339).expect("expiry")));
assert_eq!(restore_opts.mod_time, Some(old_time));
}
}
#[tokio::test]
async fn metadata_copy_honors_explicit_tag_and_expiry_replacement() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use time::format_description::well_known::Rfc3339;
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "copy-metadata-replacement";
for disk in &disks {
disk.make_volume(bucket).await.expect("create bucket");
}
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
let expires = old_time + time::Duration::days(365);
set.put_object(
bucket,
"object",
&mut PutObjReader::from_vec(b"content".to_vec()),
&ObjectOptions {
mod_time: Some(old_time),
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original");
let opts = ObjectOptions::default();
let mut source = set.get_object_info(bucket, "object", &opts).await.expect("original metadata");
source.metadata_only = true;
source.user_tags = Arc::new("old=value".to_string());
source.expires = Some(expires - time::Duration::days(1));
let metadata = Arc::make_mut(&mut source.user_defined);
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
.await
.expect("explicit metadata wins over externalized fields");
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
assert_eq!(copied.user_tags.as_str(), "new=value");
assert_eq!(copied.expires, Some(expires));
assert!(copied.mod_time.is_some_and(|time| time > old_time));
copied.metadata_only = true;
copied.user_tags = Arc::new(String::new());
copied.expires = None;
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
.await
.expect("explicit empty replacement");
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
assert!(cleared.user_tags.is_empty());
assert!(cleared.expires.is_none());
}
#[tokio::test]
async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -17011,19 +16571,6 @@ mod transition_commit_failure_tests {
.collect();
user_defined.insert("x-amz-meta-owner".to_string(), "finance".to_string());
let snowball_keys = [
"snowball-auto-extract",
"x-amz-meta-snowball-auto-extract",
"x-amz-snowball-auto-extract",
"snowball-prefix",
"x-amz-meta-rustfs-snowball-prefix",
"x-amz-meta-minio-snowball-ignore-dirs",
"other-vendor-snowball-ignore-errors",
];
for key in snowball_keys {
user_defined.insert(key.to_string(), "true".to_string());
}
let mut reader = PutObjReader::from_vec(b"stored bytes the tier keeps opaque ".repeat(64));
set_disks
.put_object(
@@ -17073,10 +16620,6 @@ mod transition_commit_failure_tests {
.await
.expect("the tier must have received the object");
for key in snowball_keys {
assert!(!remote_metadata.contains_key(key), "tier must receive opaque bytes, not {key}");
assert_eq!(transitioned.user_defined.get(key).map(String::as_str), Some("true"));
}
for (key, _) in encryption_metadata {
assert!(
!remote_metadata.keys().any(|stored| stored.eq_ignore_ascii_case(key)),
@@ -25449,131 +24992,6 @@ mod delete_objects_lock_gating_tests {
.expect("plain object should be written");
}
/// Regression for the PR9155 review finding: a pre-fetched source-missing
/// read of a later duplicate must not skip the object-lock check after an
/// earlier entry's recovery rolls back an interrupted prepared delete and
/// restores a GOVERNANCE-retained version. Without the eviction the later
/// duplicate deletes the restored version, bypassing retention.
#[tokio::test]
async fn duplicate_entries_recheck_object_lock_after_recovery_restores_retained_version() {
let (_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "recovery-retention-dup";
let object = "retained.bin";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("create bucket");
}
// A versioned put, so the batch can address the version explicitly.
let mut reader = PutObjReader::from_vec(vec![7u8; 256]);
set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
no_lock: true,
..Default::default()
},
)
.await
.expect("versioned object should be written");
let version = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("read back the version identity")
.version_id
.expect("versioned put assigns a version id");
// GOVERNANCE retention on that version, persisted the way
// PutObjectRetention persists it.
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(60);
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
no_lock: true,
version_id: Some(version.to_string()),
eval_metadata: Some(HashMap::from([
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until
.format(&time::format_description::well_known::Rfc3339)
.expect("retention date formats"),
),
])),
..Default::default()
},
)
.await
.expect("governance retention should persist");
// Interrupted authorized delete: the recoverable per-disk delete
// applies the version removal and leaves the prepared journal
// undecided — exactly a crash between apply and publish.
let transaction = Uuid::new_v4();
for disk in &disk_stores {
let fi = disk
.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
.await
.expect("version exists before the interrupted delete");
let errors = disk
.delete_versions_recoverable(
bucket,
vec![FileInfoVersions {
name: object.to_string(),
versions: vec![fi],
..Default::default()
}],
DeleteOptions {
recoverable_delete: true,
old_data_dir: Some(transaction),
..Default::default()
},
)
.await;
assert!(errors.iter().all(Option::is_none), "the plant must apply: {errors:?}");
}
// A no-bypass batch naming the retained version twice: the first
// entry rolls the prepared delete back (restoring the retained
// version) and must be denied; the second must re-read and stay
// denied too instead of reusing the stale pre-fetched miss.
let victim = ObjectToDelete {
object_name: object.to_string(),
version_id: Some(version),
..Default::default()
};
let (_deleted, errors) = set_disks
.delete_objects(
bucket,
vec![victim.clone(), victim],
ObjectOptions {
versioned: true,
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await;
assert!(
errors.iter().all(|error| error
.as_ref()
.is_some_and(|error| matches!(error, StorageError::PrefixAccessDenied(_, _)))),
"both duplicates must be denied by the restored retention: {errors:?}"
);
// The retained version survives the batch.
for disk in &disk_stores {
disk.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
.await
.expect("the retained version must survive the batch");
}
}
#[tokio::test]
#[serial_test::serial(shard_integrity_rollout)]
async fn conditional_delete_snapshot_preserves_multipart_etag() {
@@ -52,8 +52,8 @@ pub(crate) mod list {
pub(crate) mod multipart {
pub(crate) use rustfs_storage_api::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
MultipartUploadInspection, MultipartUploadResult, PartInfo,
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
MultipartUploadResult, PartInfo,
};
}
+36 -291
View File
@@ -14,7 +14,7 @@
use super::*;
use crate::bucket::{
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState},
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState, table_bucket_catalog_metadata_prefix},
utils::is_meta_bucketname,
};
use crate::error::is_err_bucket_not_found;
@@ -145,41 +145,11 @@ async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext
}
}
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
use crate::bucket::metadata::*;
let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
if bucket != "ahm-replacement" {
prefixes.push(root);
return prefixes;
}
// This valid user bucket shares its metadata directory with raw drive
// replacement intents, seals and recovery proofs. Delete only bucket-owned
// paths, including legacy standalone configuration, never the shared root.
// S3 objects live in the separate user volume; configuration APIs write the
// aggregate metadata file, so clients cannot populate arbitrary siblings.
prefixes.extend(
[
BUCKET_METADATA_FILE,
BUCKET_INCARNATION_FILE,
BUCKET_POLICY_CONFIG,
BUCKET_QUOTA_CONFIG_FILE,
BUCKET_TARGETS_FILE,
BUCKET_ACL_CONFIG,
BUCKET_TABLE_CONFIG,
BUCKET_DURABILITY_CONFIG,
BUCKET_ON_DEMAND_MIGRATION_CONFIG,
".replication",
"integrity-jobs",
crate::data_usage::DATA_USAGE_CACHE_NAME,
]
.into_iter()
.chain(XML_BUCKET_CONFIG_FILES)
.map(|file| format!("{root}/{file}")),
);
prefixes
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
[
table_bucket_catalog_metadata_prefix(bucket),
format!("{BUCKET_META_PREFIX}/{bucket}"),
]
}
fn bucket_deleted_marker_prefix(bucket: &str) -> String {
@@ -190,48 +160,41 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
}
// Box the child before constructing either lease wrapper. Keeping F inline in
// nested select futures multiplies poll-frame temporaries along DELETE -> quota
// cleanup -> metadata GET. This boundary must not inline back into that chain.
#[inline(never)]
pub(crate) fn await_bucket_namespace_operation<'a, T: 'a, F>(
guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
bucket: &'a str,
pub(crate) async fn await_bucket_namespace_operation<T, F>(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> impl Future<Output = Result<T>> + 'a
) -> Result<T>
where
F: Future<Output = Result<T>> + 'a,
F: Future<Output = Result<T>>,
{
let future = Box::pin(future);
async move {
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
let Some(guard) = guard else {
return future.await;
};
if guard.is_lock_lost() {
return Err(StorageError::other(format!(
"bucket namespace lock was lost before {operation}: {bucket}"
)));
}
tokio::select! {
biased;
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
"bucket namespace lock was lost during {operation}: {bucket}"
))),
result = future => result,
}
}
fn await_bucket_lifecycle_operation<'a, T: 'a, F>(
lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
namespace_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
bucket: &'a str,
async fn await_bucket_lifecycle_operation<T, F>(
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
operation: &'static str,
future: F,
) -> impl Future<Output = Result<T>> + 'a
) -> Result<T>
where
F: Future<Output = Result<T>> + 'a,
F: Future<Output = Result<T>>,
{
await_bucket_namespace_operation(
lifecycle_guard,
@@ -239,6 +202,7 @@ where
operation,
await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
)
.await
}
fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
@@ -514,18 +478,10 @@ impl ECStore {
let mut tags = HashMap::with_capacity(tagging.tag_set.len());
for tag in &tagging.tag_set {
let (Some(key), Some(value)) = (&tag.key, &tag.value) else {
return Err(crate::bucket::metadata::unreadable_config_error(
bucket,
BUCKET_TAGGING_CONFIG,
metadata.tagging_config_xml.len(),
));
return Err(Error::other("stored bucket tag is missing a key or value"));
};
if key.is_empty() || tags.insert(key.clone(), value.clone()).is_some() {
return Err(crate::bucket::metadata::unreadable_config_error(
bucket,
BUCKET_TAGGING_CONFIG,
metadata.tagging_config_xml.len(),
));
return Err(Error::other("stored bucket tags contain empty or duplicate keys"));
}
}
Ok(tags)
@@ -1717,17 +1673,7 @@ impl ECStore {
ns_guard.as_ref(),
bucket,
"bucket retirement publication",
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
)
.await?;
// The lifecycle fence excludes live reservation owners. A crash-left
// V1 ledger must not block quota admission after same-name recreation.
await_bucket_lifecycle_operation(
bucket_lifecycle_guard.as_ref(),
ns_guard.as_ref(),
bucket,
"retired bucket quota ledger cleanup",
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
)
.await?;
}
@@ -1878,20 +1824,6 @@ mod tests {
assert!(!xlmeta.diagnostic_truncated);
}
#[tokio::test]
async fn bucket_namespace_operation_keeps_nested_future_state_off_the_poll_stack() {
let payload = std::hint::black_box([7u8; 128 * 1024]);
let operation = super::await_bucket_lifecycle_operation(None, None, "bucket", "large metadata operation", async move {
tokio::task::yield_now().await;
Ok(std::hint::black_box(payload)[0])
});
assert!(
std::mem::size_of_val(&operation) < 1024,
"nested lease wrappers must not retain the concrete storage future inline"
);
assert_eq!(operation.await.expect("the boxed operation must still execute"), 7);
}
#[tokio::test(start_paused = true)]
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
let ttl = Duration::from_millis(20);
@@ -2308,13 +2240,7 @@ mod tests {
let result = store.get_bucket_tags_for_policy(bucket).await;
match expected {
Some(tags) => assert_eq!(&result.expect("valid tag configuration"), tags),
None => assert!(
crate::bucket::metadata::unreadable_config_refusal(
&result.expect_err("corrupt tags must not become an untagged bucket")
)
.is_some_and(|failure| failure.config_file == super::BUCKET_TAGGING_CONFIG),
"corrupt tags must carry a typed tagging refusal: {xml:?}"
),
None => assert!(result.is_err(), "corrupt tags must not become an untagged bucket: {xml:?}"),
}
}
}
@@ -2516,187 +2442,6 @@ mod tests {
assert!(prefixes.contains(&"buckets/analytics".to_string()));
}
#[tokio::test]
#[serial]
async fn replacement_named_bucket_delete_preserves_recovery_and_recreates_clean_metadata() {
use crate::bucket::metadata::{BUCKET_QUOTA_CONFIG_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
let (_temp, store) = setup_bucket_quorum_test_env(&[4], None).await;
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "ahm-replacement";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create legal bucket");
let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
for (config, bytes) in [
(BUCKET_QUOTA_CONFIG_FILE, br#"{"quota":1024,"quota_type":"Hard"}"#.as_slice()),
(
BUCKET_TAGGING_CONFIG,
b"<Tagging><TagSet><Tag><Key>old</Key><Value>value</Value></Tag></TagSet></Tagging>".as_slice(),
),
(
BUCKET_VERSIONING_CONFIG,
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".as_slice(),
),
] {
metadata_sys::update_in(&store.ctx, bucket, config, bytes.to_vec())
.await
.expect("persist old configuration");
}
let ledger_path = format!("config/quota-ledger/{bucket}.json");
// V1 precharges outstanding growth: four live bytes plus the
// crash-left reservation's 100 bytes must already be accounted.
let ledger = serde_json::to_vec(&serde_json::json!({
"version": 1, "bucket_incarnation": old_incarnation,
"quota_revision_unix_nanos": 0, "accounted_usage": 104,
"reservations": {Uuid::new_v4().to_string(): {
"object": "crashed-put", "old_size": 0, "new_size": 100,
"created_at": 0, "pool_index": 0, "set_index": 0, "commit_started": false
}}, "reconcile_required": true
}))
.expect("encode crash-left ledger");
// The storage API requires the same legacy-quota admission supplied
// by the S3 handler, even for this small fixture object.
let mut live_options = ObjectOptions {
versioned: true,
..Default::default()
};
assert!(live_options.set_quota_admission(0, 1024));
// Failed deletion must leave both the live incarnation and its ledger alone.
store
.put_object(bucket, "live", &mut PutObjReader::from_vec(b"live".to_vec()), &live_options)
.await
.expect("write live object");
crate::config::com::save_config(store.clone(), &ledger_path, ledger.clone())
.await
.expect("persist old ledger");
assert!(matches!(
store.delete_bucket(bucket, &DeleteBucketOptions::default()).await,
Err(StorageError::BucketNotEmpty(_))
));
assert_eq!(
crate::config::com::read_config(store.clone(), &ledger_path)
.await
.expect("live ledger remains"),
ledger
);
// Remove every version; the configured bucket is otherwise still empty.
let versions = store
.clone()
.list_object_versions(bucket, "", None, None, None, 100)
.await
.expect("list live versions");
for object in versions.objects {
store
.delete_object(
bucket,
&object.name,
ObjectOptions {
version_id: object.version_id.map(|id| id.to_string()),
..Default::default()
},
)
.await
.expect("remove live version");
}
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
let task = Uuid::new_v4();
let recovery_names = [
"ahm_replacement_intent.json",
"ahm_replacement_intent_seal",
"ahm_replacement_completion_proof.json",
]
.map(|suffix| format!("{task}_{suffix}"));
let disk_paths = store.pools[0].disk_set[0]
.disks
.read()
.await
.iter()
.flatten()
.map(|disk| PathBuf::from(disk.endpoint().to_string()))
.collect::<Vec<_>>();
for disk in &disk_paths {
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
for name in &recovery_names {
tokio::fs::write(directory.join(name), b"durable heal record")
.await
.expect("seed raw recovery record");
}
// Legacy standalone settings and bucket-owned subtrees must not survive.
for name in [
"quota.json",
"versioning.xml",
"policy.json",
"bucket-targets.json",
"integrity-jobs/job.json",
".replication/resync.bin",
".usage-cache.bin",
] {
let path = directory.join(name);
tokio::fs::create_dir_all(path.parent().expect("fixture parent"))
.await
.expect("legacy parent");
tokio::fs::write(path, b"old bucket configuration")
.await
.expect("seed legacy configuration");
}
}
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete legal same-name bucket");
assert!(matches!(
crate::config::com::read_config(store.clone(), &ledger_path).await,
Err(StorageError::ConfigNotFound)
));
for disk in &disk_paths {
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
for name in &recovery_names {
assert_eq!(
tokio::fs::read(directory.join(name)).await.expect("recovery record survives"),
b"durable heal record"
);
}
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket)
.into_iter()
.filter(|p| p.starts_with(&root))
{
assert!(
!disk.join(RUSTFS_META_BUCKET).join(prefix).exists(),
"bucket-owned metadata must be removed"
);
}
}
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate legal bucket");
let fresh = metadata_sys::get_in(&store.ctx, bucket).await.expect("fresh metadata");
assert_ne!(fresh.bucket_incarnation_id, old_incarnation);
assert!(fresh.quota_config_json.is_empty());
assert!(fresh.tagging_config_xml.is_empty());
assert!(fresh.versioning_config_xml.is_empty());
assert!(fresh.policy_config_json.is_empty());
// The strict purge/recovery cleanup must share the same ownership boundary.
store
.cleanup_deleted_bucket_metadata_strict(bucket, true)
.await
.expect("strict metadata cleanup");
for disk in &disk_paths {
for name in &recovery_names {
assert_eq!(
tokio::fs::read(disk.join(RUSTFS_META_BUCKET).join(&root).join(name))
.await
.expect("strict cleanup preserves recovery"),
b"durable heal record"
);
}
}
}
#[test]
fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
+5 -311
View File
@@ -368,7 +368,7 @@ impl ECStore {
Some(expected) => expected,
None => self.bucket_incarnation_id_from_disk(bucket).await?,
};
let object = object.map(encode_dir_object);
let object = object.map(str::to_owned);
let prefix = prefix.map(str::to_owned);
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
@@ -430,7 +430,7 @@ impl ECStore {
HealItemType::Bucket
}
.to_string(),
object: object.map(|object| decode_dir_object(&object)).or(prefix).unwrap_or_default(),
object: object.or(prefix).unwrap_or_default(),
bucket,
detail: format!(
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
@@ -656,48 +656,12 @@ impl ECStore {
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
}
/// Read the published bucket identity without fabricating or migrating metadata.
/// Absence is distinct from quorum, corruption and transaction-fence failures.
pub async fn replacement_bucket_incarnation_from_disk(self: &Arc<Self>, bucket: &str) -> Result<Option<uuid::Uuid>> {
let transaction =
crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
let incarnation = crate::bucket::metadata::load_bucket_incarnation(self.clone(), bucket).await?;
if transaction.is_lock_lost() {
return Err(Error::PreconditionFailed);
}
Ok(incarnation)
}
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
pub async fn heal_replacement_bucket_metadata(
self: &Arc<Self>,
bucket: &str,
opts: &HealOpts,
targets: &[String],
) -> Result<()> {
// Reject an invalid request before touching the drives so callers keep
// the PreconditionFailed contract for bad buckets and options.
if opts.pool.is_none()
|| opts.set.is_none()
|| targets.is_empty()
|| opts.dry_run
|| opts.no_lock
|| check_valid_bucket_name_strict(bucket).is_err()
{
return Err(Error::PreconditionFailed);
}
let expected = self.bucket_incarnation_id_from_disk(bucket).await?;
self.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
.await
}
/// Keep a replacement pass bound to its admitted bucket through metadata commit.
pub async fn heal_replacement_bucket_metadata_at_incarnation(
self: &Arc<Self>,
bucket: &str,
expected: uuid::Uuid,
opts: &HealOpts,
targets: &[String],
) -> Result<()> {
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
@@ -734,8 +698,9 @@ impl ECStore {
// Read the persisted identity without lazy migration: a replacement
// must never turn lost configuration into newly fabricated defaults.
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
let targets = targets.to_vec();
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, opts| async move {
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
// Match config writers: lifecycle, metadata transaction, object locks.
// Keep the transaction stable through target readback, including when
// the caller is cancelled and the storage owner finishes its write.
@@ -1057,7 +1022,6 @@ impl ECStore {
set_index: usize,
targets: &[String],
) -> Result<bool> {
let object = encode_dir_object(object);
let pool = self
.pools
.get(pool_index)
@@ -1070,7 +1034,7 @@ impl ECStore {
)
})?;
set.replacement_targets_have_version(bucket, &object, version_id, targets)
set.replacement_targets_have_version(bucket, object, version_id, targets)
.await
.map_err(Into::into)
}
@@ -2895,105 +2859,6 @@ mod tests {
}
}
#[tokio::test]
#[serial_test::serial]
async fn replacement_directory_object_readback_and_exact_recovery_use_encoded_key() {
let (_root, store, shutdown) = heal_store_with_pool_count(1).await;
let bucket = "directory-replacement-readback";
let object = "folder/";
let encoded = encode_dir_object(object);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
store
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(Vec::new()),
&ObjectOptions {
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write directory object through the public store entry");
let set = &store.pools[0].disk_set[0];
let targets = vec![set.set_endpoints[0].to_string()];
assert!(
store
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
.await
.expect("readback")
);
let disk = set.disks.read().await[0].clone().expect("target disk");
disk.delete(
bucket,
&encoded,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("remove target directory shard");
assert!(
!store
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
.await
.expect("missing target")
);
let opts = HealOpts {
pool: Some(0),
set: Some(0),
..Default::default()
};
let (result, error) = store
.heal_object(bucket, object, "", &opts)
.await
.expect("heal directory object");
assert!(error.is_none(), "heal failed: {error:?}");
assert_eq!(result.object, object);
assert!(
store
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
.await
.expect("healed readback")
);
assert!(
store
.replacement_targets_have_version(bucket, &encoded, "", 0, 0, &targets)
.await
.expect("encoded readback")
);
// Exact-object recovery must inspect the same physical namespace and
// reject corrupt proof rather than silently report no transactions.
let metadata = disk
.read_all(bucket, &format!("{encoded}/xl.meta"))
.await
.expect("healed metadata");
disk.write_all(
bucket,
&format!("{encoded}/{}/delete-prepared", Uuid::new_v4()),
b"invalid proof".to_vec().into(),
)
.await
.expect("seed corrupt recovery proof");
store
.recover_delete_transactions_at_incarnation(bucket, Some(object), None, &HealOpts { dry_run: true, ..opts })
.await
.expect_err("directory recovery must discover and reject the corrupt proof");
assert_eq!(
disk.read_all(bucket, &format!("{encoded}/xl.meta"))
.await
.expect("unchanged metadata"),
metadata
);
shutdown.cancel();
}
#[tokio::test]
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
let (root, store, shutdown) = multi_pool_heal_store().await;
@@ -3827,177 +3692,6 @@ mod tests {
);
}
// The RPC lock service uses the raw ObjectKey and one backend per node,
// even when the requesting sets carry different pool namespace strings.
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
use rustfs_lock::client::{LockClient, local::LocalClient};
let clients = || {
(0..2)
.map(|_| {
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
})
.collect::<Vec<_>>()
};
let first = clients();
let second = if shared { first.clone() } else { clients() };
let mut pools = source.pools.clone();
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
for set in &mut Arc::make_mut(pool).disk_set {
Arc::make_mut(set).lockers = lockers.clone();
}
}
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
Arc::new(ECStore {
id: Uuid::new_v4(),
disk_map: Default::default(),
pools,
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
rebalance_meta: tokio::sync::RwLock::new(None),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(state),
ctx: Arc::clone(&source.ctx),
bucket_fence_registry: Arc::default(),
})
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
let first_lock = first
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("first pool lock");
let second_lock = second
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("second pool lock");
let guard = first_lock
.get_write_lock(std::time::Duration::from_secs(2))
.await
.expect("first writer");
assert!(
second_lock
.get_write_lock(std::time::Duration::from_millis(150))
.await
.is_err(),
"different pool namespaces still conflict on the same backend and object key"
);
drop(guard);
second_lock
.get_write_lock(std::time::Duration::from_secs(3))
.await
.expect("second writer should acquire after the first releases");
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let (item, error) = tokio::time::timeout(
std::time::Duration::from_secs(3),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
dry_run: true,
..Default::default()
},
),
)
.await
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
.expect("pool metadata heal inspection should return a result");
assert!(error.is_none(), "inspection should succeed: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let missing = remove_pool_meta_shard(&store, 1).await;
let (item, error) = tokio::time::timeout(
std::time::Duration::from_secs(10),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
..Default::default()
},
),
)
.await
.expect("cross-pool repair must not reenter the shared writer")
.expect("pool metadata repair should return a result");
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
assert!(
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
"the selected missing shard must be restored"
);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
let lock = target
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("target lock");
let guard = lock
.get_write_lock(std::time::Duration::from_secs(2))
.await
.expect("hold target lock");
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
let heal_store = Arc::clone(&store);
let mut heal = tokio::spawn(async move {
heal_store
.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
dry_run: true,
..Default::default()
},
)
.await
});
tokio::time::timeout(
std::time::Duration::from_secs(3),
barrier.wait_until_external_heal_target_lock_attempted(),
)
.await
.expect("inspection must attempt the independent target lock");
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
drop(guard);
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
.await
.expect("inspection should progress after target lock release")
.expect("heal task should not panic")
.expect("pool metadata heal inspection should return a result");
assert!(error.is_none(), "inspection should succeed: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
-298
View File
@@ -5155,11 +5155,6 @@ mod tests {
let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
if !sharded {
crate::bucket::quota::reservation::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
.await
.expect("bootstrap quota with one disk offline");
}
let payload = vec![0x51; 4096];
let mut reader = PutObjReader::from_vec(payload.clone());
@@ -21012,299 +21007,6 @@ mod tests {
);
}
async fn seed_retained_null_version(
store: &Arc<crate::store::ECStore>,
bucket: &str,
object: &str,
protection: &str,
latest_is_marker: bool,
) {
let metadata = if protection == "legal-hold" {
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
} else {
HashMap::from([
("x-amz-object-lock-mode".to_string(), protection.to_string()),
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
])
};
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
store
.put_object(
bucket,
object,
&mut original,
&ObjectOptions {
user_defined: metadata,
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
..Default::default()
},
)
.await
.expect("seed retained null version");
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
store
.put_object(
bucket,
object,
&mut latest,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed newer unprotected UUID version");
if latest_is_marker {
store
.delete_object(
bucket,
object,
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("create a UUID delete marker above the retained null version");
}
}
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
let mut reader = store
.get_object_reader(
bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
},
)
.await
.expect("retained null version must remain readable");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("read retained null version");
assert_eq!(body, b"protected null version");
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn null_version_object_lock_delete_checks_actual_target() {
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
let temp = tempfile::tempdir().expect("create null-version delete store");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("create Object Lock bucket");
let configs = [
VersioningConfiguration::default(),
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
];
for (case, config) in configs.into_iter().enumerate() {
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
for latest_is_marker in [false, true] {
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
let (_, errors) = store
.delete_objects(
&bucket,
vec![ObjectToDelete {
object_name: object.clone(),
..Default::default()
}],
ObjectOptions {
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
..Default::default()
},
)
.await;
assert!(
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
"batch delete must judge the retained null version: {object}: {errors:?}"
);
let error = store
.delete_object(
&bucket,
&object,
ObjectOptions {
version_suspended: true,
..Default::default()
},
)
.await
.expect_err("suspended single delete must also judge the retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
assert_retained_null_version_body(&store, &bucket, &object).await;
if protection == "GOVERNANCE" {
let (_, errors) = store
.delete_objects(
&bucket,
vec![ObjectToDelete {
object_name: object.clone(),
..Default::default()
}],
ObjectOptions {
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
bypass_governance: true,
}),
..Default::default()
},
)
.await;
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
let removed = store
.get_object_info(
&bucket,
&object,
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
},
)
.await;
match removed {
Ok(marker) => {
assert!(marker.delete_marker, "only a null delete marker may replace the payload");
assert_eq!(marker.size, 0);
assert!(marker.data_dir.is_none());
}
Err(error) => assert!(
matches!(
error,
StorageError::MethodNotAllowed
| StorageError::ObjectNotFound(_, _)
| StorageError::VersionNotFound(_, _, _)
),
"unexpected bypass result: {error:?}"
),
}
assert!(
store
.get_object_reader(
&bucket,
&object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
}
)
.await
.is_err(),
"authorized deletion must make the retained null payload unreadable"
);
}
}
}
}
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
let temp = tempfile::tempdir().expect("create null-version overwrite store");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("create Object Lock bucket");
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
for latest_is_marker in [false, true] {
let object = format!("{protection}-{latest_is_marker}");
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
let error = store
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
.await
.expect_err("PUT must reject replacement of a retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
let upload = store
.clone()
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
.await
.expect("initiate multipart upload");
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
let part = store
.clone()
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
.await
.expect("stage multipart replacement");
let error = store
.clone()
.complete_multipart_upload(
&bucket,
&object,
&upload.upload_id,
vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
.expect_err("multipart completion must reject replacement of a retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
assert_retained_null_version_body(&store, &bucket, &object).await;
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
store
.put_object(
&bucket,
&object,
&mut next_version,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("a new UUID version must remain allowed");
assert_retained_null_version_body(&store, &bucket, &object).await;
}
}
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
File diff suppressed because it is too large Load Diff
+2
View File
@@ -423,6 +423,8 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
!is_meta_bucketname(&oi.bucket)
}
const MAX_UPLOADS_LIST: usize = 10000;
mod bucket;
mod bucket_fence;
pub(crate) use bucket::await_bucket_namespace_operation;
+15 -422
View File
@@ -273,28 +273,6 @@ async fn pause_data_movement_multipart_before_selected_completion(bucket: &str)
}
}
async fn pool_has_multipart_upload_for_key(
pool: &crate::core::sets::Sets,
bucket: &str,
object: &str,
expected_incarnation_id: Option<Uuid>,
) -> Result<bool> {
let results = stream::iter(pool.disk_set.iter().cloned())
.map(|set| async move {
set.has_multipart_upload_for_key(bucket, object, expected_incarnation_id)
.await
})
.buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY)
.collect::<Vec<_>>()
.await;
let mut found = false;
for result in results {
// Preserve errors from every set instead of hiding them behind a hit.
found |= result?;
}
Ok(found)
}
async fn list_pool_multipart_uploads_for_incarnation(
pool: &crate::core::sets::Sets,
bucket: &str,
@@ -512,8 +490,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ListPartsInfo> {
check_list_parts_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
@@ -644,8 +620,6 @@ impl ECStore {
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
check_new_object_args(bucket, object, opts.data_movement)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if self.single_pool() {
@@ -701,7 +675,21 @@ impl ECStore {
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
continue;
}
if pool_has_multipart_upload_for_key(pool, bucket, object, opts.expected_bucket_incarnation_id).await? {
let res = list_pool_multipart_uploads_for_incarnation(
pool,
bucket,
&MultipartUploadListRequest {
prefix: object.to_string(),
key_marker: None,
upload_id_marker: None,
delimiter: None,
max_uploads: MAX_UPLOADS_LIST,
expected_incarnation_id: opts.expected_bucket_incarnation_id,
},
)
.await?;
if !res.uploads.is_empty() {
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
@@ -784,8 +772,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<PartInfo> {
check_put_object_part_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if let Some(guard) = bucket_lifecycle_guard {
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
@@ -820,8 +806,6 @@ impl ECStore {
.part_number
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
check_put_object_part_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
if !opts.data_movement {
return Err(Error::other("targeted multipart upload requires data_movement options"));
}
@@ -854,8 +838,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<MultipartInfo> {
check_list_parts_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
if self.single_pool() {
@@ -889,8 +871,6 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<()> {
check_abort_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
for pool_idx in self.existing_multipart_pool_order().await {
@@ -1072,8 +1052,6 @@ impl ECStore {
for upload_id in upload_ids {
check_abort_multipart_args(bucket, object, upload_id)?;
}
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
if !opts.data_movement {
return Err(Error::other("targeted multipart abort requires data_movement options"));
}
@@ -1232,34 +1210,7 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
check_complete_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
// The set-level commit judges the null version this completion
// replaces; give it the same authoritative snapshot a PUT carries.
let expected_incarnation_id = opts
.expected_bucket_incarnation_id
.ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
let lifecycle_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
Some(snapshot) => Arc::clone(snapshot),
None => {
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
.await?
}
};
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
return Err(Error::other(
"Object Lock snapshot does not hold valid target bucket generation and configuration fences",
));
}
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
if !self.single_pool() {
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
}
@@ -1368,8 +1319,6 @@ impl ECStore {
let request_opts = opts;
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
if !opts.data_movement {
return Err(Error::other("targeted multipart completion requires data_movement options"));
}
@@ -1491,362 +1440,6 @@ mod tests {
};
use std::time::Duration;
#[tokio::test]
async fn create_multipart_pool_selection_uses_only_an_exact_key_upload() {
use crate::bucket::metadata_sys::init_bucket_metadata_sys;
use crate::runtime::instance::InstanceContext;
let mut dirs = Vec::new();
let mut pools = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let directory = tempfile::tempdir().expect("temporary disk");
let mut endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("local endpoint");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
dirs.push(directory);
endpoints.push(endpoint);
}
pools.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("exact-multipart-pool-{pool_index}"),
platform: "test".to_owned(),
});
}
let endpoints = EndpointServerPools(pools);
let ctx = Arc::new(InstanceContext::new());
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
.await
.expect("initialize local disks");
let cancel = tokio_util::sync::CancellationToken::new();
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, cancel.clone(), ctx)
.await
.expect("initialize two pools");
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "exact-pool-selection";
let other_bucket = "other-pool-selection";
for bucket in [bucket, other_bucket] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
}
for (upload_bucket, key) in [(bucket, "report-long"), (bucket, "report/"), (other_bucket, "report")] {
let (opts, _guard) = store
.guard_multipart_bucket_incarnation(upload_bucket, &ObjectOptions::default())
.await
.expect("pin upload incarnation");
store.pools[0]
.new_multipart_upload(upload_bucket, &encode_dir_object(key), &opts)
.await
.expect("seed nonmatching upload in first pool");
}
let (opts, _guard) = store
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
.await
.expect("pin target incarnation");
assert!(
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", opts.expected_bucket_incarnation_id)
.await
.expect("first pool contains only other keys")
);
store.pools[1]
.new_multipart_upload(bucket, "report", &opts)
.await
.expect("seed exact upload in second pool");
let (created, pool_index, incarnation) = store
.handle_new_multipart_upload_with_pool_idx(bucket, "report", &ObjectOptions::default(), None)
.await
.expect("create another upload for the exact key");
assert_eq!(pool_index, 1, "prefix matches in the first pool must not change placement");
assert_eq!(incarnation, opts.expected_bucket_incarnation_id);
assert!(!created.upload_id.is_empty());
assert!(
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", incarnation)
.await
.expect("new upload must not appear in first pool")
);
assert!(
!pool_has_multipart_upload_for_key(&store.pools[1], bucket, "report", Some(Uuid::new_v4()))
.await
.expect("old bucket incarnations must not affect placement")
);
cancel.cancel();
}
#[tokio::test]
async fn multipart_directory_keys_preserve_the_bare_object_and_round_trip() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::object::ObjectIO as _;
use sha2::Digest;
use tokio::io::AsyncReadExt;
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
for versioned in [false, true] {
let bucket = if versioned {
"directory-mpu-versioned"
} else {
"directory-mpu-unversioned"
};
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
if versioned {
store
.update_bucket_metadata_config(
bucket,
"versioning.xml",
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec(),
)
.await
.expect("enable versioning");
}
let opts = ObjectOptions {
versioned,
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
};
let original = store
.put_object(
bucket,
"protected",
&mut PutObjReader::from_vec(b"private original".to_vec()),
&ObjectOptions {
user_defined: HashMap::from([(
rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(),
"classification=secret".to_string(),
)]),
..opts.clone()
},
)
.await
.expect("write protected bare object");
let upload = store
.new_multipart_upload(bucket, "protected/", &opts)
.await
.expect("create directory upload");
let part = store
.put_object_part(
bucket,
"protected/",
&upload.upload_id,
1,
&mut PutObjReader::from_vec(b"directory payload".to_vec()),
&opts,
)
.await
.expect("write directory part");
store
.put_object_part(
bucket,
"protected/",
&upload.upload_id,
2,
&mut PutObjReader::from_vec(b"unselected private bytes".to_vec()),
&opts,
)
.await
.expect("write an unselected part");
let upload_sha = hex_simd::encode_to_string(
sha2::Sha256::digest(format!("{bucket}/protected__XLDIR__").as_bytes()),
hex_simd::AsciiCase::Lower,
);
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
for dir in &dirs {
let staging = dir
.path()
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_sha)
.join(&upload_uuid);
let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(staging.join("xl.meta")).expect("upload metadata"))
.expect("decode upload metadata");
let versions = metadata
.get_file_info_versions(bucket, "protected__XLDIR__", true)
.expect("upload file info");
let data_dir = versions
.versions
.first()
.expect("upload version")
.data_dir
.expect("staging data directory");
assert!(
staging.join(data_dir.to_string()).join("part.2").is_file(),
"unselected part exists before completion"
);
}
assert_eq!(
store
.get_multipart_info(bucket, "protected/", &upload.upload_id, &opts)
.await
.expect("upload info")
.object,
"protected/"
);
assert_eq!(
store
.list_object_parts(bucket, "protected/", &upload.upload_id, None, 10, &opts)
.await
.expect("list parts")
.object,
"protected/"
);
let uploads = store
.list_multipart_uploads(bucket, "protected/", None, None, None, 10)
.await
.expect("list logical prefix");
assert_eq!(uploads.uploads.len(), 1);
assert_eq!(uploads.uploads[0].object, "protected/");
assert_eq!(crate::runtime::sources::upload_uuid_suffix(&uploads.uploads[0].upload_id), upload_uuid,);
let completed = store
.clone()
.complete_multipart_upload(
bucket,
"protected/",
&upload.upload_id,
vec![CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&opts,
)
.await
.expect("complete directory upload");
assert_eq!(completed.name, "protected/");
for (key, expected) in [
("protected", b"private original".as_slice()),
("protected/", b"directory payload".as_slice()),
] {
let mut reader = store
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read distinct object");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("read body");
assert_eq!(bytes, expected);
}
let preserved = store
.get_object_info(bucket, "protected", &ObjectOptions::default())
.await
.expect("bare metadata");
assert_eq!(preserved.version_id, original.version_id);
assert_eq!(preserved.etag, original.etag);
assert_eq!(preserved.user_tags.as_str(), "classification=secret");
for dir in &dirs {
assert!(dir.path().join(bucket).join("protected/xl.meta").is_file());
assert!(dir.path().join(bucket).join("protected__XLDIR__/xl.meta").is_file());
let data = dir
.path()
.join(bucket)
.join("protected__XLDIR__")
.join(completed.data_dir.expect("multipart data directory").to_string());
assert!(data.join("part.1").is_file());
assert!(!data.join("part.2").exists(), "unselected part must never be published");
assert!(
!dir.path()
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
.join(&upload_sha)
.join(&upload_uuid)
.exists(),
"original staging must be cleaned"
);
}
let abandoned = store
.new_multipart_upload(bucket, "protected/", &opts)
.await
.expect("create abort control");
store
.abort_multipart_upload(bucket, "protected/", &abandoned.upload_id, &opts)
.await
.expect("abort encoded upload");
assert!(
store
.get_multipart_info(bucket, "protected/", &abandoned.upload_id, &opts)
.await
.is_err()
);
assert!(
store
.get_object_info(bucket, "protected/", &ObjectOptions::default())
.await
.is_ok()
);
}
}
#[cfg(not(windows))]
#[tokio::test]
async fn new_control_character_keys_are_rejected_without_trapping_legacy_objects() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::object::ObjectIO as _;
use tokio::io::AsyncReadExt;
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "xml-key-admission";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
let key = format!("bad-{}", char::from(byte));
assert!(
store
.put_object(bucket, &key, &mut PutObjReader::from_vec(b"body".to_vec()), &ObjectOptions::default())
.await
.is_err()
);
assert!(
store
.new_multipart_upload(bucket, &key, &ObjectOptions::default())
.await
.is_err()
);
for dir in &dirs {
assert!(!dir.path().join(bucket).join(&key).exists());
}
}
let legacy_key = "legacy-\u{1}";
store.pools[0]
.get_disks_by_key(legacy_key)
.put_object(
bucket,
legacy_key,
&mut PutObjReader::from_vec(b"legacy payload".to_vec()),
&ObjectOptions {
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("seed a pre-upgrade object below new-key admission");
let mut reader = store
.get_object_reader(bucket, legacy_key, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("legacy GET stays usable");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("legacy bytes");
assert_eq!(bytes, b"legacy payload");
drop(reader);
store
.delete_object(bucket, legacy_key, ObjectOptions::default())
.await
.expect("legacy DELETE stays usable");
assert!(
store
.get_object_info(bucket, legacy_key, &ObjectOptions::default())
.await
.is_err()
);
}
fn mp(object: &str, upload_id: &str) -> MultipartInfo {
MultipartInfo {
bucket: "bucket".to_string(),
File diff suppressed because it is too large Load Diff
@@ -20,230 +20,14 @@ use std::sync::Arc;
use std::time::Duration;
use storage_api::contract_compat::ECStore;
use storage_api::metadata_lock::{
BucketOperations, CompletePart, DeleteBucketOptions, Error, HashReader, MakeBucketOptions, MultipartOperations,
NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause,
init_bucket_metadata_sys, isolated_store_over_temp_disks,
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
isolated_store_over_temp_disks,
};
use tokio::io::AsyncReadExt;
use tokio::time::timeout;
use uuid::Uuid;
struct ObservedPutBody {
inner: tokio::io::DuplexStream,
polled: Arc<tokio::sync::Notify>,
}
impl tokio::io::AsyncRead for ObservedPutBody {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
self.polled.notify_one();
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
#[tokio::test]
async fn destructive_put_takes_bucket_locks_only_after_its_body_is_staged() {
use tokio::io::AsyncWriteExt;
for (object, version_id) in [("object", None), ("directory/", Some(Uuid::nil().to_string()))] {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "deferred-put-locks";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let encoded_object = if object.ends_with('/') { "directory__XLDIR__" } else { object };
let barrier = PutObjectCommitBarrier::install(bucket, encoded_object, PutObjectCommitPause::BeforeQuotaRename);
let (mut sender, inner) = tokio::io::duplex(16);
let polled = Arc::new(tokio::sync::Notify::new());
let body = ObservedPutBody {
inner,
polled: polled.clone(),
};
let put_store = store.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
put_store
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
version_id,
..Default::default()
},
)
.await
});
timeout(Duration::from_secs(5), polled.notified())
.await
.expect("PUT must be waiting for client data");
let lock = store
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
.await
.expect("metadata lock");
drop(
timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5)))
.await
.expect("body must not block metadata writers")
.expect("metadata write guard"),
);
drop(
timeout(Duration::from_secs(2), store.lock_bucket_for_recursive_delete(bucket))
.await
.expect("body must not retain lifecycle guard")
.expect("lifecycle writer"),
);
sender.write_all(b"body").await.expect("finish body");
sender.shutdown().await.expect("EOF");
timeout(Duration::from_secs(5), barrier.wait_until_paused())
.await
.expect("staged PUT reaches commit");
let writer = lock.get_write_lock(Duration::from_secs(5));
tokio::pin!(writer);
assert!(futures::poll!(&mut writer).is_pending(), "commit must fence metadata updates");
let lifecycle = store.lock_bucket_for_recursive_delete(bucket);
tokio::pin!(lifecycle);
assert!(futures::poll!(&mut lifecycle).is_pending(), "commit must fence bucket deletion");
barrier.release();
let release_metadata = async {
drop(writer.await.expect("metadata writer"));
};
let release_lifecycle = async {
drop(lifecycle.await.expect("lifecycle writer"));
};
timeout(Duration::from_secs(5), async {
let (put, (), ()) = tokio::join!(put, release_metadata, release_lifecycle);
put.expect("owner task").expect("commit");
})
.await
.expect("PUT and queued writers finish");
}
}
#[tokio::test]
async fn staged_put_cannot_publish_into_a_recreated_bucket() {
use tokio::io::AsyncWriteExt;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "deferred-put-generation";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let (mut sender, inner) = tokio::io::duplex(16);
let polled = Arc::new(tokio::sync::Notify::new());
let body = ObservedPutBody {
inner,
polled: polled.clone(),
};
let put_store = store.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
put_store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
});
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
store
.delete_bucket(
bucket,
&DeleteBucketOptions {
force: true,
..Default::default()
},
)
.await
.expect("remove old bucket while PUT stalls");
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
sender.write_all(b"body").await.expect("finish body");
sender.shutdown().await.expect("EOF");
let error = timeout(Duration::from_secs(5), put)
.await
.expect("commit gate")
.expect("owner")
.expect_err("old request generation must not publish");
assert!(
error.to_string().contains("generation") || matches!(error, Error::BucketNotFound(..)),
"{error:?}"
);
assert!(
store
.get_object_info(bucket, "object", &ObjectOptions::default())
.await
.is_err()
);
}
#[tokio::test]
async fn staged_put_rechecks_current_object_lock_at_commit() {
use tokio::io::AsyncWriteExt;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "deferred-put-retention";
store
.make_bucket(
bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("locked bucket");
let mut old = PutObjReader::from_vec(b"old".to_vec());
store
.put_object(bucket, "object", &mut old, &ObjectOptions::default())
.await
.expect("unlocked predecessor");
let (mut sender, inner) = tokio::io::duplex(16);
let polled = Arc::new(tokio::sync::Notify::new());
let body = ObservedPutBody {
inner,
polled: polled.clone(),
};
let put_store = store.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
put_store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
});
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
let mut protected = PutObjReader::from_vec(b"protected".to_vec());
store
.put_object(
bucket,
"object",
&mut protected,
&ObjectOptions {
user_defined: std::collections::HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
..Default::default()
},
)
.await
.expect("install a protected predecessor while the client stalls");
sender.write_all(b"body").await.expect("finish stale body");
sender.shutdown().await.expect("EOF");
let error = timeout(Duration::from_secs(5), put)
.await
.expect("commit gate")
.expect("task")
.expect_err("pre-body state cannot admit an overwrite of the newly protected version");
assert!(matches!(error, Error::PrefixAccessDenied(..)), "{error:?}");
let info = store
.get_object_info(bucket, "object", &ObjectOptions::default())
.await
.expect("protected object remains");
assert_eq!(info.size, 9);
}
#[tokio::test]
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
replica_write_with_waiting_metadata_writer(false).await;
+1 -3
View File
@@ -9,14 +9,12 @@ use rustfs_storage_api as storage_contracts;
#[cfg(feature = "test-util")]
pub(crate) mod metadata_lock {
pub(crate) use super::storage_contracts::{
BucketOperations, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO,
ObjectOperations,
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
};
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
pub(crate) use rustfs_ecstore::api::rio::HashReader;
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
}
-28
View File
@@ -3930,34 +3930,6 @@ mod test {
assert!(versions_with_free.versions.iter().all(|version| version.num_versions == 2));
}
#[test]
fn add_version_rejects_mod_times_that_cannot_round_trip() {
for nanos in [0, i128::from(i64::MAX) + 1, i128::from(i64::MIN) - 1, 1_i128 << 64] {
for deleted in [false, true] {
let mut meta = FileMeta::new();
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
info.deleted = deleted;
info.mod_time = Some(OffsetDateTime::from_unix_timestamp_nanos(nanos).expect("time within supported date range"));
assert!(
meta.add_version(info).is_err(),
"unrepresentable time {nanos} must be rejected before publication"
);
}
}
for nanos in [-1, 1, i64::MIN, i64::MAX] {
let mut meta = FileMeta::new();
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
let time = OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).expect("valid codec time");
info.mod_time = Some(time);
meta.add_version(info).expect("representable times remain supported");
let reopened = FileMeta::load(&meta.marshal_msg().expect("serialize version")).expect("reopen version");
let versions = reopened
.get_file_info_versions("bucket", "object", false)
.expect("read version history");
assert_eq!(versions.versions[0].mod_time, Some(time));
}
}
#[test]
fn test_data_integrity_validation() {
// Test data integrity checks
+2 -13
View File
@@ -858,24 +858,13 @@ impl FileMetaVersion {
VersionType::Object => self
.object
.as_ref()
.map(|v| {
v.erasure_algorithm.valid()
&& v.bitrot_checksum_algo.valid()
&& v.mod_time.is_some_and(|time| {
let nanos = time.unix_timestamp_nanos();
nanos != 0 && i64::try_from(nanos).is_ok()
})
})
.map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some())
.unwrap_or_default(),
VersionType::Legacy => self.legacy_object.as_ref().map(MetaObjectV1::valid).unwrap_or_default(),
VersionType::Delete => self
.delete_marker
.as_ref()
.map(|v| {
v.mod_time.is_some_and(|time| {
time > OffsetDateTime::UNIX_EPOCH && i64::try_from(time.unix_timestamp_nanos()).is_ok()
})
})
.map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH)
.unwrap_or_default(),
_ => false,
}
+2 -81
View File
@@ -925,13 +925,6 @@ impl MetaCacheEntriesSorted {
const METACACHE_STREAM_VERSION: u8 = 2;
const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
/// Upper bound on one incremental reservation while accumulating a single
/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
/// up to this bound so a legitimately large xl.meta does not pay one small
/// read per chunk, while a corrupt length prefix still cannot force a
/// reservation larger than the payload that has already arrived (see
/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
/// msgpack `false`: the entry-stream terminator, and the only byte a legacy
/// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -1164,14 +1157,13 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
// A whole xl.meta can legitimately exceed the limit on its individual
// msgpack values. Grow only as payload arrives, so a corrupt length
// prefix cannot cause a speculative allocation of the advertised size.
let mut chunk = METACACHE_READ_CHUNK_SIZE;
while self.buf.len() < read_size {
let start = self.buf.len();
let extra = if lookahead {
let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
(read_size - start).clamp(min_chunk, chunk)
(read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
} else {
(read_size - start).min(chunk)
(read_size - start).min(METACACHE_READ_CHUNK_SIZE)
};
if let Err(e) = self.buf.try_reserve(extra) {
let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1202,7 +1194,6 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
return Err(err);
}
}
chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
}
self.pos = read_size;
@@ -2164,76 +2155,6 @@ mod tests {
}
}
/// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
/// decode completely even when the aggregate stream exceeds the per-value
/// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
/// element.
#[tokio::test]
async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
// ~2.9 KiB of metadata per entry matches the per-entry cost observed in
// the #9120 production report; 6,000 entries push the stream past 16 MiB.
const ENTRIES: usize = 6_000;
let metadata = vec![0xab; 2_880];
let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
.map(|i| MetaCacheEntry {
name: format!("bigdir/object-{i:05}"),
metadata: metadata.clone(),
..Default::default()
})
.collect();
let mut output = Cursor::new(Vec::new());
let mut writer = MetacacheWriter::new(&mut output);
writer.write(&entries).await.expect("wide listing must encode");
writer.close().await.expect("stream must close");
let bytes = output.into_inner();
assert!(
bytes.len() > 16 << 20,
"fixture must exceed the per-value limit, got {} bytes",
bytes.len()
);
let mut reader = MetacacheReader::new(Cursor::new(bytes));
let decoded = reader.read_all().await.expect("wide listing must decode");
assert_eq!(decoded, entries);
}
/// The #9120 failure shape seen in production: one entry whose metadata is
/// an xl.meta that grew past 16 MiB with its version history, listed among
/// ordinary neighbors and skipped past without losing the framing.
#[tokio::test]
async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
let small = |i: u8| MetaCacheEntry {
name: format!("bigdir/object-{i}"),
metadata: vec![i; 64],
..Default::default()
};
let oversized = MetaCacheEntry {
name: "bigdir/huge-versions".to_string(),
metadata: vec![0xcd; (16 << 20) + 12_345],
..Default::default()
};
let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
let mut output = Cursor::new(Vec::new());
let mut writer = MetacacheWriter::new(&mut output);
writer.write(&entries).await.expect("mixed listing must encode");
writer.close().await.expect("stream must close");
let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
assert_eq!(
reader.peek().await.expect("first entry").expect("present"),
entries[0],
"an ordinary entry before the oversized one must decode"
);
reader.skip(1).await.expect("skip the first entry");
assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
assert_eq!(
reader.peek().await.expect("oversized entry").expect("present"),
oversized,
"an entry past the per-value limit must round-trip whole"
);
reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
}
#[test]
fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
let version_id = Uuid::new_v4();
+3 -41
View File
@@ -77,9 +77,6 @@ pub enum Error {
#[error("Replacement target is not ready: {0}")]
ReplacementTargetNotReady(String),
#[error("Replacement bucket namespace changed; a fresh identity-bound pass is required")]
ReplacementNamespaceChanged,
#[error("replacement recovery retry budget exhausted")]
ReplacementRetryBudgetExhausted,
@@ -131,23 +128,6 @@ impl Error {
Error::TransientSkip { message: message.into() }
}
/// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
/// Callers must cut their own concurrency instead of retrying at full speed.
pub(crate) fn is_slow_down(&self) -> bool {
match self {
Error::Storage(error) => matches!(
error,
EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles | EcstoreError::DeleteRecoveryWorkLimit
),
Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
message.contains("Please reduce your request rate")
}
Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
_ => false,
}
}
/// Whether a heal operation can be retried without changing its inputs.
pub(crate) fn is_recoverable_heal(&self) -> bool {
match classify_heal_error(self) {
@@ -235,10 +215,9 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
Error::DanglingDeleteDeferred { retry_not_before } => HealErrorClass::DanglingGrace {
retry_not_before: Some(*retry_not_before),
},
Error::ReplacementTargetNotReady(_)
| Error::ReplacementNamespaceChanged
| Error::TransientSkip { .. }
| Error::BucketDiscoveryIncomplete => HealErrorClass::Transient,
Error::ReplacementTargetNotReady(_) | Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => {
HealErrorClass::Transient
}
Error::LegacyMrfUnverified { .. } => HealErrorClass::Permanent,
Error::Storage(error) => classify_storage_error(error),
Error::Disk(error) => classify_disk_error(error),
@@ -292,7 +271,6 @@ pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
| EcstoreError::FaultyDisk
| EcstoreError::FaultyRemoteDisk
| EcstoreError::SlowDown
| EcstoreError::DeleteRecoveryWorkLimit
| EcstoreError::OperationCanceled
| EcstoreError::RemoteClientUnavailable(_) => HealErrorClass::Transient,
_ if error.is_quorum_error() => HealErrorClass::Transient,
@@ -471,22 +449,6 @@ mod tests {
use super::Error;
use crate::heal::{DiskError, EcstoreError};
#[test]
fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
assert!(
Error::TaskExecutionFailed {
message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
}
.is_slow_down()
);
assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
}
#[test]
fn pool_metadata_failures_keep_their_typed_recovery_classification() {
use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
+119 -240
View File
@@ -14,7 +14,6 @@
use crate::error::{HealErrorClass, classify_heal_error};
use crate::heal::{
pacing::HealPageRate,
progress::{HealProgress, add_bytes, increment_counter},
resume::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
@@ -129,12 +128,12 @@ pub struct ErasureSetHealer {
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
work_parent: Option<super::manager::HealWork>,
mainline_pacer: Option<super::pacing::ForegroundPacing>,
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
}
async fn acquire_page_permit(
semaphore: Arc<Semaphore>,
rate: &HealPageRate,
pacer: Option<&super::pacing::MainlinePacer>,
cancel: &tokio_util::sync::CancellationToken,
) -> Result<tokio::sync::OwnedSemaphorePermit> {
let acquire = || async {
@@ -147,16 +146,18 @@ async fn acquire_page_permit(
let mut paid_pause = false;
loop {
let permit = acquire().await?;
// Keep the real permit on the low-pressure path. Every acquisition
// gets a fresh decision, including a waiter that queued a second
// time. One completed pause is a bounded minimum-progress grant.
match rate.admission_decision() {
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
drop(permit);
paid_pause = rate.wait_after_admission(cancel, pressure).await?;
continue;
if let Some(pacer) = pacer {
// Keep the real permit on the low-pressure path. Every acquisition
// gets a fresh decision, including a waiter that queued a second
// time. One completed pause is a bounded minimum-progress grant.
match pacer.admission_decision() {
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
drop(permit);
paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
continue;
}
_ => {}
}
_ => {}
}
return Ok(permit);
}
@@ -165,7 +166,7 @@ async fn acquire_page_permit(
#[cfg(test)]
mod mainline_pacing_tests {
use super::*;
use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
use crate::heal::pacing::{MainlinePacer, TestPressure};
use rustfs_concurrency::WorkloadClass;
use std::sync::atomic::Ordering;
use tokio_util::sync::CancellationToken;
@@ -179,15 +180,15 @@ mod mainline_pacing_tests {
.await
.expect("existing object owns capacity");
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
let pacer =
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
let pacer = Arc::new(
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
);
let cancel = CancellationToken::new();
let waiting = tokio::spawn({
let semaphore = semaphore.clone();
let rate = rate.clone();
let pacer = pacer.clone();
let cancel = cancel.clone();
async move { acquire_page_permit(semaphore, &rate, &cancel).await }
async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
});
tokio::task::yield_now().await;
provider.active.store(100, Ordering::SeqCst);
@@ -199,13 +200,12 @@ mod mainline_pacing_tests {
assert_eq!(semaphore.available_permits(), 1);
let deadline = tokio::time::timeout(
Duration::from_millis(10),
acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
)
.await;
assert!(deadline.is_err());
assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
let unpaced = HealPageRate::new(1, None);
let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
.await
.expect("unpaced admission");
assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
@@ -221,11 +221,10 @@ mod mainline_pacing_tests {
let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
let pause = Duration::from_millis(250);
let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
let rate = HealPageRate::new(1, Some(pacer));
let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
let cancel = CancellationToken::new();
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
assert!(futures::poll!(first.as_mut()).is_pending());
assert!(futures::poll!(second.as_mut()).is_pending());
drop(occupied);
@@ -330,20 +329,6 @@ impl ErasureSetHealer {
}
}
/// Page fan-out for one whole erasure-set pass (stable across buckets).
fn pass_page_concurrency_limit(&self) -> usize {
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
// Versions of the same object must not race without its namespace lock.
1
} else {
Self::effective_heal_page_object_concurrency_for_source(
self.source,
self.heal_opts.scan_mode,
self.replacement_task_id.is_some(),
)
}
}
/// Map the shared heal error class onto this healer's per-object actions.
///
/// `classify_heal_error` is the only classifier; this adapter decides what
@@ -388,7 +373,7 @@ impl ErasureSetHealer {
}
}
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
self.mainline_pacer = pacer;
self
}
@@ -468,16 +453,7 @@ impl ErasureSetHealer {
let task_id = self.get_or_create_task_id(set_disk_id).await?;
// 2. initialize or resume resume state
let (resume_manager, mut checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
let replacement_buckets = if self.replacement_task_id.is_some() {
let current = resume_manager.prepare_replacement_bucket_pass(self.storage.as_ref()).await?;
// Preparation may have durably rewound the companion checkpoint.
checkpoint_manager = CheckpointManager::load_from_disk(self.disk.clone(), &task_id).await?;
Some(current)
} else {
None
};
let buckets = replacement_buckets.as_deref().unwrap_or(buckets);
let (resume_manager, checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
let admin_buckets = if self.admin_task.is_some() {
Some(
@@ -488,7 +464,12 @@ impl ErasureSetHealer {
None
};
let buckets = admin_buckets.as_deref().unwrap_or(buckets);
if checkpoint_manager.admin_completed().await {
if checkpoint_manager
.get_checkpoint()
.await
.admin
.is_some_and(|admin| admin.completed)
{
self.initialize_progress(buckets, &resume_manager.get_state().await).await;
return Ok(());
}
@@ -498,9 +479,6 @@ impl ErasureSetHealer {
.execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
.await;
if self.replacement_task_id.is_some() {
resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
}
result?;
self.verify_replacement_identity_fence("completion").await?;
@@ -512,7 +490,14 @@ impl ErasureSetHealer {
}
if self.admin_task.is_some() {
checkpoint_manager.complete_admin().await?;
let mut admin = checkpoint_manager
.get_checkpoint()
.await
.admin
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
admin.completed = true;
admin.outcome.finish(None);
checkpoint_manager.set_admin(admin).await?;
} else {
checkpoint_manager.cleanup().await?;
resume_manager.cleanup().await?;
@@ -733,10 +718,7 @@ impl ErasureSetHealer {
}
return Err(Error::DanglingDeleteDeferred { retry_not_before });
}
// One rate controller for the whole pass: SlowDown cool-down and page
// fan-out persist across buckets instead of resetting at each one.
let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
debug!(
target: "rustfs::heal::erasure_healer",
@@ -744,17 +726,14 @@ impl ErasureSetHealer {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
set_disk_id,
current_bucket_index,
current_object_index,
current_bucket_index = checkpoint.current_bucket_index,
current_object_index = checkpoint.current_object_index,
state = "resuming",
"Erasure set resumed"
);
// 2. initialize progress
self.initialize_progress(buckets, &state).await;
// Snapshot before any baseline write: set_progress_baseline overwrites
// the saved generation, which would hide a real generation mismatch.
let checkpoint = checkpoint_manager.progress_snapshot().await;
let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
let baseline = self.progress.read().await;
(
@@ -781,7 +760,17 @@ impl ErasureSetHealer {
let mut successful_objects = state.successful_objects;
let mut failed_objects = state.failed_objects;
let mut skipped_objects = state.skipped_objects;
let checkpoint_has_progress = checkpoint.has_progress();
let checkpoint_has_progress = checkpoint.baseline_known
|| checkpoint.successful_objects > 0
|| checkpoint.failed_object_count > 0
|| checkpoint.skipped_object_count > 0
|| checkpoint.skipped_new_versions > 0
|| checkpoint.skipped_ilm_expired > 0
|| checkpoint.processed_bytes > 0
|| checkpoint.total_objects > 0
|| checkpoint.total_bytes > 0
|| checkpoint.baseline_generation.is_some()
|| checkpoint.counter_unknown;
let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
if checkpoint_has_progress {
@@ -865,7 +854,6 @@ impl ErasureSetHealer {
resume_manager,
checkpoint_manager,
state.start_time,
&page_rate,
)
.await;
@@ -979,16 +967,9 @@ impl ErasureSetHealer {
// Recheck even resumed buckets: their user-object cursor does not
// prove that the replacement holds the internal bucket records.
for bucket in buckets {
if self.replacement_task_id.is_some() {
let expected = resume_manager.replacement_bucket_incarnation(bucket).await?;
self.storage
.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, &self.heal_opts, targets)
.await?;
} else {
self.storage
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
.await?;
}
self.storage
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
.await?;
}
}
self.heal_replacement_pool_metadata(
@@ -1038,16 +1019,15 @@ impl ErasureSetHealer {
// never hot-retried in place here.
if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
if self.replacement_task_id.is_some() {
resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
let state = resume_manager.get_state().await;
let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
if state.can_retry()
&& failed_objects == 0
&& failed_buckets == 0
&& skipped_objects > 0
&& skipped_objects == grace_objects
&& !counter_unknown
&& let Some(retry_not_before) = retry_not_before
&& skipped_objects == checkpoint.dangling_delete_grace_objects
&& !checkpoint.counter_unknown
&& let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
{
// Publish the generation's rewind and wait first. If the
// checkpoint reset is interrupted, initialization reapplies
@@ -1187,9 +1167,14 @@ impl ErasureSetHealer {
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
let checkpoint_key = compose_key(&object_key, None);
if checkpoint_manager.is_object_settled(&checkpoint_key).await {
let checkpoint = checkpoint_manager.get_checkpoint().await;
if checkpoint.processed_objects.contains(&checkpoint_key)
|| checkpoint.failed_objects.contains(&checkpoint_key)
|| checkpoint.skipped_objects.contains(&checkpoint_key)
{
return Ok(());
}
drop(checkpoint);
self.verify_replacement_identity_fence("pool metadata").await?;
resume_manager
@@ -1443,7 +1428,7 @@ impl ErasureSetHealer {
/// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
async fn heal_bucket_with_resume(
&self,
bucket: &str,
@@ -1457,7 +1442,6 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
started_at_secs: u64,
page_rate: &Arc<HealPageRate>,
) -> Result<bool> {
debug!(
target: "rustfs::heal::erasure_healer",
@@ -1472,11 +1456,7 @@ impl ErasureSetHealer {
"Erasure set bucket started"
);
let incarnation = if self.replacement_task_id.is_some() {
Some(resume_manager.replacement_bucket_incarnation(bucket).await?)
} else {
self.admin_bucket_incarnation(checkpoint_manager, bucket).await?
};
let incarnation = self.admin_bucket_incarnation(checkpoint_manager, bucket).await?;
let unresolved_delete_recovery = self.heal_admin_bucket_delete_recovery(bucket, incarnation).await?;
if unresolved_delete_recovery && !self.heal_opts.dry_run {
resume_manager.set_delete_recovery_rescan_pending(true).await?;
@@ -1511,6 +1491,17 @@ impl ErasureSetHealer {
// detects a non-advancing cursor even though next_marker embeds a
// transient cache id that changes between identical pages.
let mut previous_page_last: Option<(String, Option<String>)> = None;
let page_concurrency_limit =
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
// Versions of the same object must not race without its namespace lock.
1
} else {
Self::effective_heal_page_object_concurrency_for_source(
self.source,
self.heal_opts.scan_mode,
self.replacement_task_id.is_some(),
)
};
let in_flight = Arc::new(AtomicUsize::new(0));
// Per-bucket sample caps for per-object warn! lines: a flapping rebuild
// disk can fail/skip hundreds of thousands of versions in one sweep, so
@@ -1532,7 +1523,9 @@ impl ErasureSetHealer {
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
loop {
page_rate.wait_before_page(&self.cancel_token).await?;
if let Some(pacer) = &self.mainline_pacer {
pacer.wait(&self.cancel_token).await?;
}
self.verify_replacement_identity_fence("page scan").await?;
if let Some(expected) = incarnation {
self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
@@ -1554,8 +1547,9 @@ impl ErasureSetHealer {
.await?
};
let page_is_empty = objects.is_empty();
let checkpoint = checkpoint_manager.get_checkpoint().await;
let page_resume_index = *current_object_index;
let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
let mut page_tasks = FuturesUnordered::new();
let mut completed_in_page = 0usize;
@@ -1569,7 +1563,10 @@ impl ErasureSetHealer {
// Per-version dedup identity — the single canonical key.
let key = compose_key(&item.name, item.version_id.as_deref());
if checkpoint_manager.is_object_settled(&key).await {
if checkpoint.processed_objects.contains(&key)
|| checkpoint.failed_objects.contains(&key)
|| checkpoint.skipped_objects.contains(&key)
{
continue;
}
@@ -1727,7 +1724,7 @@ impl ErasureSetHealer {
let target_endpoints = self.target_endpoints.clone();
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
let page_rate = page_rate.clone();
let mainline_pacer = self.mainline_pacer.clone();
let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
@@ -1739,15 +1736,14 @@ impl ErasureSetHealer {
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
let work = async move {
let mut retries = 0;
let mut saw_slow_down = false;
let mut admin_identity = admin_identity;
let mut admin_task_id = admin_task_id;
loop {
let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
let permit = match permit {
Ok(permit) => permit,
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
};
let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
@@ -1755,8 +1751,7 @@ impl ErasureSetHealer {
if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
let (result, canonical) =
admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
return (dedup_key, object_name, version_id, result, canonical);
}
// Always go through heal_object. Genuine absence flows through
@@ -1768,20 +1763,15 @@ impl ErasureSetHealer {
(0, Err(Error::TaskCancelled))
} else {
let result = if replacement_object {
match incarnation {
Some(expected) => storage
.heal_replacement_object_at_incarnation(
&bucket_name,
&object_name,
version_id.as_deref(),
expected,
&heal_opts,
&cancel_token,
)
.await
.map(|result| (result.item, result.error)),
None => Err(Error::ReplacementNamespaceChanged),
}
storage
.heal_object_with_task_cancel(
&bucket_name,
&object_name,
version_id.as_deref(),
&heal_opts,
&cancel_token,
)
.await
} else {
storage
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
@@ -1863,23 +1853,18 @@ impl ErasureSetHealer {
drop(in_flight_guard);
drop(permit);
if matches!(&result.1, Err(err) if err.is_slow_down()) {
saw_slow_down = true;
}
if retry_replacement
&& retries < REPLACEMENT_OBJECT_MAX_RETRIES
&& matches!(&result.1, Err(Error::TransientSkip { .. }))
{
retries += 1;
tokio::select! {
_ = cancel_token.cancelled() => {
return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
}
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
_ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
}
continue;
}
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
return (dedup_key, object_name, version_id, result, None);
}
};
page_tasks.push(async move {
@@ -1890,7 +1875,7 @@ impl ErasureSetHealer {
};
match result {
Ok(result) => result,
Err(error) => (key, object, version, (0, Err(error), false), None),
Err(error) => (key, object, version, (0, Err(error)), None),
}
} else {
work.await
@@ -1899,16 +1884,7 @@ impl ErasureSetHealer {
}
while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
let (object_size, result, saw_slow_down) = result;
// Exactly one rate observation per finished object, including
// SlowDown that later succeeded on a replacement retry.
page_rate.observe_object_outcome(
&match &result {
Ok(_) => Ok(true),
Err(err) => Err(err),
},
saw_slow_down,
);
let (object_size, result) = result;
let mut telemetry_unknown = false;
let checkpoint_outcome = match result {
Ok(true) => {
@@ -2506,7 +2482,6 @@ mod resume_loop_tests {
use super::{
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
};
use crate::heal::pacing::HealPageRate;
use crate::heal::progress::HealProgress;
use crate::heal::resume::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
@@ -2667,7 +2642,6 @@ mod resume_loop_tests {
struct FakeStorage {
admin_disk: Mutex<Option<DiskStore>>,
bucket_discovery_incomplete: AtomicBool,
bucket_discovery_error: Mutex<Option<EcstoreError>>,
bucket_discovery_scopes: Mutex<Vec<Option<crate::heal::storage_api::storage::DiskSetSelector>>>,
bucket_pages: Mutex<HashMap<String, Page>>,
incarnations: Mutex<HashMap<String, uuid::Uuid>>,
@@ -2759,9 +2733,6 @@ mod resume_loop_tests {
if self.bucket_discovery_incomplete.load(Ordering::SeqCst) {
return Err(Error::BucketDiscoveryIncomplete);
}
if let Some(error) = self.bucket_discovery_error.lock().expect("discovery failure").clone() {
return Err(Error::Storage(error));
}
self.list_buckets().await
}
@@ -2938,16 +2909,6 @@ mod resume_loop_tests {
}
Ok(())
}
async fn heal_replacement_bucket_metadata_at_incarnation(
&self,
bucket: &str,
expected: uuid::Uuid,
opts: &HealOpts,
targets: &[String],
) -> Result<()> {
self.validate_bucket_incarnation(bucket, Some(expected)).await?;
self.heal_replacement_bucket_metadata(bucket, opts, targets).await
}
async fn replacement_targets_have_version(
&self,
_bucket: &str,
@@ -3064,16 +3025,6 @@ mod resume_loop_tests {
let temp = TempDir::new().unwrap();
let disk = make_disk(&temp).await;
let storage = Arc::new(FakeStorage::default());
if !target_endpoints.is_empty() {
storage.bucket_pages.lock().expect("replacement inventory").insert(
"b".to_string(),
Page {
items: Vec::new(),
next: None,
truncated: false,
},
);
}
let task_id = ResumeUtils::generate_task_id();
let healer = ErasureSetHealer::new(
storage.clone(),
@@ -3093,17 +3044,7 @@ mod resume_loop_tests {
)
.await
.unwrap();
if !healer.target_endpoints.is_empty() {
resume
.prepare_replacement_bucket_pass(storage.as_ref())
.await
.expect("replacement fixture identity admission");
}
let checkpoint = if CheckpointManager::has_checkpoint(&disk, &task_id).await {
CheckpointManager::load_from_disk(disk, &task_id).await.unwrap()
} else {
CheckpointManager::new(disk, task_id.clone()).await.unwrap()
};
let checkpoint = CheckpointManager::new(disk, task_id.clone()).await.unwrap();
Env {
healer,
storage,
@@ -3122,7 +3063,6 @@ mod resume_loop_tests {
let mut successful = 0u64;
let mut failed = 0u64;
let mut skipped = 0u64;
let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
let result = env
.healer
.heal_bucket_with_resume(
@@ -3137,66 +3077,11 @@ mod resume_loop_tests {
&env.resume,
&env.checkpoint,
state.start_time,
&page_rate,
)
.await;
(processed, successful, failed, skipped, result.map(|_| ()))
}
#[tokio::test]
async fn replacement_bucket_pass_rejects_incomplete_inventory_without_reusing_or_clearing_progress() {
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
env.resume
.set_resume_cursor(Some("held-cursor".to_string()))
.await
.expect("cursor before discovery failure");
env.checkpoint
.add_processed_object("held-key".to_string())
.await
.expect("dedup before discovery failure");
// Model a page whose dedup identities must survive until cursor
// publication. complete_page deliberately prunes that identity set.
env.checkpoint
.advance_page(0, 7)
.await
.expect("persist the retained page identities");
let before = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.expect("durable checkpoint before discovery failure")
.get_checkpoint()
.await;
assert_eq!(before.current_object_index, 7);
assert!(before.processed_objects.contains("held-key"));
let state = env.resume.get_state().await;
env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst);
assert!(matches!(
env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
Err(Error::BucketDiscoveryIncomplete)
));
assert_eq!(env.resume.get_state().await, state, "partial discovery must never publish a fresh plan");
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.expect("durable checkpoint")
.get_checkpoint()
.await;
assert_eq!(checkpoint.current_object_index, 7);
assert!(checkpoint.processed_objects.contains("held-key"));
env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst);
*env.storage.bucket_discovery_error.lock().expect("inject quorum failure") =
Some(EcstoreError::InsufficientReadQuorum("bucket".to_string(), "inventory".to_string()));
assert!(matches!(
env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
Err(Error::Storage(EcstoreError::InsufficientReadQuorum(..)))
));
assert_eq!(env.resume.get_state().await, state, "quorum failure is not an empty namespace");
*env.storage.bucket_discovery_error.lock().expect("restore inventory") = None;
env.resume
.prepare_replacement_bucket_pass(env.storage.as_ref())
.await
.expect("unchanged complete inventory resumes existing pass");
assert_eq!(env.resume.get_state().await.resume_cursor.as_deref(), Some("held-cursor"));
}
#[tokio::test]
async fn test_empty_bucket_no_panic() {
let env = make_env().await;
@@ -3259,7 +3144,6 @@ mod resume_loop_tests {
let mut skipped = 0;
let started_at = env.resume.get_state().await.start_time;
let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
let error = healer
.heal_bucket_with_resume(
"b",
@@ -3273,7 +3157,6 @@ mod resume_loop_tests {
&env.resume,
&env.checkpoint,
started_at,
&page_rate,
)
.await
.expect_err("a remounted target must not begin a new page scan");
@@ -4172,7 +4055,7 @@ mod resume_loop_tests {
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
],
async {
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer = env
.healer
@@ -4227,7 +4110,7 @@ mod resume_loop_tests {
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
],
async {
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer.heal_opts.no_lock = true;
env.healer = env
@@ -4284,7 +4167,7 @@ mod resume_loop_tests {
#[tokio::test]
async fn auto_replacement_retries_a_transient_object_before_rewinding_the_scan() {
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer = env
.healer
@@ -4316,7 +4199,7 @@ mod resume_loop_tests {
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("1")),
],
async {
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer = env
.healer
@@ -4348,7 +4231,7 @@ mod resume_loop_tests {
#[tokio::test]
async fn auto_replacement_cancellation_interrupts_object_retry() {
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer = env
.healer
@@ -4364,18 +4247,14 @@ mod resume_loop_tests {
env.storage.set_outcome("retry", None, HealOutcome::Transient);
let cancel = env.healer.cancel_token.clone();
let observe = async {
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
let notified = env.storage.heal_started.notified();
if !env.storage.calls().is_empty() {
break;
}
notified.await;
loop {
let notified = env.storage.heal_started.notified();
if !env.storage.calls().is_empty() {
break;
}
})
.await;
notified.await;
}
cancel.cancel();
started.expect("replacement object retry must start");
};
let ((_, successful, _, _, result), ()) = tokio::join!(run(&env), observe);
@@ -4393,7 +4272,7 @@ mod resume_loop_tests {
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
],
async {
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
let execution = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
env.healer = env
@@ -51,25 +51,6 @@ async fn replacement_env() -> Env {
)
.await
.expect("replacement intent should persist before scanning");
env.resume
.prepare_replacement_bucket_pass(env.storage.as_ref())
.await
.expect("the new replacement generation must admit its bucket identities before direct execution");
// Admission durably resets the companion checkpoint. Reload its current
// revision instead of retaining the previous generation's CAS instance.
env.checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.expect("reload the admitted replacement pass checkpoint");
assert_eq!(
env.resume
.replacement_bucket_incarnation("b")
.await
.expect("the fixture bucket must belong to the replacement pass"),
env.storage
.admit_bucket_incarnation("b")
.await
.expect("fixture bucket identity")
);
env.healer = ErasureSetHealer::new(
env.storage.clone(),
Arc::new(RwLock::new(HealProgress::new())),
+1 -8
View File
@@ -810,14 +810,7 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
active_heals
.iter()
.find(|(_, task)| {
PriorityHealQueue::make_dedup_key_for_owner(
&task.heal_type,
&task.options,
task.source,
task.expected_mrf_bucket_incarnation_id,
) == key
})
.find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
}
+1 -98
View File
@@ -611,104 +611,7 @@ impl PriorityHealQueue {
/// Create a deduplication key from a heal request
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
Self::make_dedup_key_for_owner(
&request.heal_type,
&request.options,
request.source,
request.expected_mrf_bucket_incarnation_id,
)
}
/// Bound MRF work can only merge with work for the same bucket lifetime.
/// Length-prefix user-controlled components so a key cannot forge another
/// version, scope, or incarnation through delimiter characters.
pub(super) fn make_dedup_key_for_owner(
heal_type: &HealType,
options: &HealOptions,
source: HealRequestSource,
incarnation: Option<Uuid>,
) -> String {
if source == HealRequestSource::Mrf
&& let Some(incarnation) = incarnation
{
use std::fmt::Write as _;
let (pool, set) = heal_scope_indices(heal_type, options);
let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
let mut part = |value: &str| {
// Formatting into a String is infallible.
let _ = write!(key, "{}:{value}", value.len());
};
match heal_type {
HealType::Object {
bucket,
object,
version_id,
} => {
part("object");
part(bucket);
part(object);
match version_id {
None => part("none"),
Some(version_id) => {
part("some");
part(version_id);
}
}
}
HealType::Metadata { bucket, object } => {
part("metadata");
part(bucket);
part(object);
}
HealType::ECDecode {
bucket,
object,
version_id,
} => {
part("ecdecode");
part(bucket);
part(object);
match version_id {
None => part("none"),
Some(version_id) => {
part("some");
part(version_id);
}
}
}
HealType::DeleteMarkerPurge {
bucket,
object,
version_id,
purge,
} => {
part("delete-marker-purge");
part(bucket);
part(object);
part(version_id);
part(&purge.bucket_incarnation_id.to_string());
part(&purge.marker_incarnation_id.to_string());
part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
}
HealType::Bucket { bucket } => {
part("bucket");
part(bucket);
}
HealType::Prefix { bucket, prefix } => {
part("prefix");
part(bucket);
part(prefix);
}
// Cluster and erasure-set work are not bucket-bound; preserve
// their existing keys, including contains_erasure_set lookups.
HealType::Cluster | HealType::ErasureSet { .. } => {
return Self::make_dedup_key_for_scope(heal_type, options);
}
}
return key;
}
Self::make_dedup_key_for_scope(heal_type, options)
Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
}
pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {
+3 -2
View File
@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
request: &HealRequest,
config: &HealConfig,
provider: &Option<WorkloadSnapshotProviderRef>,
) -> Option<crate::heal::pacing::ForegroundPacing> {
) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
let pace_running = request.source == HealRequestSource::Admin
|| (request.source == HealRequestSource::AutoHeal
&& matches!(&request.heal_type, HealType::ErasureSet { .. })
@@ -43,13 +43,14 @@ pub(super) fn running_mainline_pacer(
return None;
}
provider.as_ref().and_then(|provider| {
crate::heal::pacing::ForegroundPacing::new(
crate::heal::pacing::MainlinePacer::new(
provider.clone(),
config.mainline_read_utilization_high_percent,
config.mainline_write_utilization_high_percent,
config.mainline_max_sleep,
request.source,
)
.map(Arc::new)
})
}
-447
View File
@@ -117,453 +117,6 @@ async fn process_manager_queue_once(manager: &HealManager) {
struct MockStorage;
fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
request.source = HealRequestSource::Mrf;
request.expected_mrf_bucket_incarnation_id = Some(incarnation);
request.options.pool_index = Some(0);
request.options.set_index = Some(0);
request
}
async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
let old_incarnation = Uuid::new_v4();
let new_incarnation = Uuid::new_v4();
let old_request = incarnation_bound_mrf_fallback(old_incarnation);
let old_id = old_request.id.clone();
match existing_state {
"queued" => {
let receipt = manager
.submit_heal_request_with_receipt(old_request)
.await
.expect("old fallback must be admitted");
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
}
"active" => {
manager
.active_heals
.lock()
.await
.insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
}
"retrying" => {
manager.retrying_heals.lock().await.insert(
old_id.clone(),
RetryingHeal {
request: old_request,
error: "retry pending".to_owned(),
cancel_token: CancellationToken::new(),
},
);
}
other => panic!("unknown fixture state: {other}"),
}
let new_request = incarnation_bound_mrf_fallback(new_incarnation);
let new_id = new_request.id.clone();
assert_ne!(new_id, old_id, "the fallback must use a new request ID");
let receipt = manager
.submit_heal_request_with_receipt(new_request)
.await
.expect("new incarnation admission must be decided");
assert_eq!(
receipt.result,
HealAdmissionResult::Accepted,
"a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
);
assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
let queue = manager.heal_queue.lock().await;
assert!(
queue
.requests()
.any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
);
if existing_state == "queued" {
assert!(
queue
.requests()
.any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
);
}
}
#[tokio::test]
async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
}
#[tokio::test]
async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
assert_recreated_bucket_fallback_has_its_own_owner("active").await;
}
#[tokio::test]
async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
}
fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
let mut request = incarnation_bound_mrf_fallback(incarnation);
if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
*id = version_id.map(str::to_owned);
} else {
unreachable!("fallback fixture must be an object request");
}
request
}
#[tokio::test]
async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
for version in [Uuid::nil(), Uuid::new_v4()] {
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let incarnation = Uuid::new_v4();
let version_id = version.to_string();
let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
let original_id = original.id.clone();
let accepted = manager
.submit_heal_request_with_receipt(original)
.await
.expect("first queued fallback");
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
let merged = manager
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
.await
.expect("duplicate queued fallback");
assert_eq!(merged.result, HealAdmissionResult::Merged);
assert_eq!(merged.task_id, original_id);
assert_eq!(manager.heal_queue.lock().await.len(), 1);
}
}
#[tokio::test]
async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let incarnation = Uuid::new_v4();
let original = incarnation_bound_mrf_fallback(incarnation);
let original_id = original.id.clone();
assert_eq!(
manager
.submit_heal_request_with_receipt(original)
.await
.expect("first fallback")
.result,
HealAdmissionResult::Accepted
);
let same = manager
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
.await
.expect("same-incarnation fallback");
assert_eq!(same.result, HealAdmissionResult::Merged);
assert_eq!(same.task_id, original_id);
let mut versioned = incarnation_bound_mrf_fallback(incarnation);
versioned.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: "object".to_owned(),
version_id: Some(Uuid::new_v4().to_string()),
};
assert_eq!(
manager
.submit_heal_request_with_receipt(versioned)
.await
.expect("different version")
.result,
HealAdmissionResult::Accepted
);
let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
other_scope.options.pool_index = Some(1);
assert_eq!(
manager
.submit_heal_request_with_receipt(other_scope)
.await
.expect("different pool")
.result,
HealAdmissionResult::Accepted
);
let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
same_id_other_incarnation.id = original_id;
assert_eq!(
manager
.submit_heal_request_with_receipt(same_id_other_incarnation)
.await
.expect("same-ID conflict must be decided")
.result,
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
}
#[test]
fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
let incarnation = Uuid::new_v4();
let bound = incarnation_bound_mrf_fallback(incarnation);
let mut unbound = bound.clone();
unbound.expected_mrf_bucket_incarnation_id = None;
let mut non_mrf = bound.clone();
non_mrf.source = HealRequestSource::ReadRepair;
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
assert_eq!(
PriorityHealQueue::make_dedup_key(&non_mrf),
PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
"an unrelated source retains its previous scope key even if the MRF-only field is set"
);
assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
let mut left = bound.clone();
left.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: "part:a".to_owned(),
version_id: Some("b".to_owned()),
};
let mut right = bound.clone();
right.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: "part".to_owned(),
version_id: Some("a:b".to_owned()),
};
assert_eq!(
PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
"the legacy delimiter-only key demonstrates the collision"
);
assert_ne!(
PriorityHealQueue::make_dedup_key(&left),
PriorityHealQueue::make_dedup_key(&right),
"length-prefixed object and version components must not alias"
);
let mut forged = unbound;
forged.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: PriorityHealQueue::make_dedup_key(&bound),
version_id: None,
};
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
}
#[test]
fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
let incarnation = Uuid::new_v4();
for decode in [false, true] {
let make_request = |object: &str, version_id: Option<&str>| {
let mut request = incarnation_bound_mrf_fallback(incarnation);
let bucket = "mrf-recreated-bucket".to_owned();
let object = object.to_owned();
let version_id = version_id.map(str::to_owned);
request.heal_type = if decode {
HealType::ECDecode {
bucket,
object,
version_id,
}
} else {
HealType::Object {
bucket,
object,
version_id,
}
};
request
};
let absent = make_request("object", None);
let empty = make_request("object", Some(""));
let forged_object = make_request("object4:some0:", None);
let forged_version = make_request("object", Some("4:none"));
let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
for (index, key) in keys.iter().enumerate() {
assert!(keys.iter().skip(index + 1).all(|other| key != other));
}
let mut queue = PriorityHealQueue::new();
assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
assert_eq!(queue.len(), 2);
for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
let mut legacy_absent = absent.clone();
let mut legacy_empty = empty.clone();
legacy_absent.source = source;
legacy_empty.source = source;
if source == HealRequestSource::Mrf {
legacy_absent.expected_mrf_bucket_incarnation_id = None;
legacy_empty.expected_mrf_bucket_incarnation_id = None;
}
assert_eq!(
PriorityHealQueue::make_dedup_key(&legacy_absent),
PriorityHealQueue::make_dedup_key(&legacy_empty),
"unbound and non-MRF work retain the legacy empty-version key"
);
}
}
}
#[test]
fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
let mut queue = PriorityHealQueue::new();
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
let old_id = old.id.clone();
let old_key = PriorityHealQueue::make_dedup_key(&old);
let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
let new_id = new.id.clone();
let new_key = PriorityHealQueue::make_dedup_key(&new);
let mut forced_old = old.clone();
forced_old.id = Uuid::new_v4().to_string();
forced_old.force_start = true;
let forced_id = forced_old.id.clone();
assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
queue
.remove_request_id(&old_id)
.expect("remove the old incarnation representative");
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
}
#[tokio::test]
async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
assert_eq!(
manager
.submit_heal_request_with_receipt(old)
.await
.expect("old fallback")
.result,
HealAdmissionResult::Accepted
);
let incarnation = Uuid::new_v4();
let request = incarnation_bound_mrf_fallback(incarnation);
let request_id = request.id.clone();
let mut intent = MrfIntent {
bucket: Arc::from("mrf-recreated-bucket"),
object: Arc::from("object"),
version_id: None,
kind: MrfKind::PartialWrite,
delete_marker_purge: None,
scope: Some(MrfScope {
pool_index: 0,
set_index: 0,
}),
lease: None,
enqueued_at_ms: 1,
attempts: 0,
};
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
let target = MrfRepairNoticeTarget {
bucket: Arc::clone(&intent.bucket),
object: Arc::clone(&intent.object),
version_id: intent.version_id,
kind: intent.kind,
scope: intent.scope,
delete_marker_purge: None,
lease: intent.lease,
durable_anchor: Some(anchor.clone()),
};
let receipt = manager
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
.await
.expect("durable admission must be decided");
assert_eq!(
receipt.result,
HealAdmissionResult::Accepted,
"a durable new incarnation needs its own owner"
);
assert_eq!(receipt.task_id, request_id);
assert!(
manager
.heal_queue
.lock()
.await
.requests()
.any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
);
assert!(
manager
.try_in_flight_durable_mrf_anchors()
.expect("stable owner snapshot")
.contains(&anchor)
);
}
#[tokio::test]
async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let incarnation = Uuid::new_v4();
let object = format!("same-generation-{}", Uuid::new_v4());
let make_request = || {
let mut request = incarnation_bound_mrf_fallback(incarnation);
request.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: object.clone(),
version_id: None,
};
request
};
let old = make_request();
let old_id = old.id.clone();
assert_eq!(
manager
.submit_heal_request_with_receipt(old)
.await
.expect("old fallback")
.result,
HealAdmissionResult::Accepted
);
let mut intent = MrfIntent {
bucket: Arc::from("mrf-recreated-bucket"),
object: Arc::from(object.as_str()),
version_id: None,
kind: MrfKind::PartialWrite,
delete_marker_purge: None,
scope: Some(MrfScope {
pool_index: 0,
set_index: 0,
}),
lease: None,
enqueued_at_ms: 1,
attempts: 0,
};
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
let target = MrfRepairNoticeTarget {
bucket: Arc::clone(&intent.bucket),
object: Arc::clone(&intent.object),
version_id: intent.version_id,
kind: intent.kind,
scope: intent.scope,
delete_marker_purge: None,
lease: intent.lease,
durable_anchor: Some(anchor.clone()),
};
let merged = manager
.submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
.await
.expect("durable same-incarnation admission");
assert_eq!(merged.result, HealAdmissionResult::Merged);
assert_eq!(merged.task_id, old_id);
assert!(
manager
.try_in_flight_durable_mrf_anchors()
.expect("stable owner snapshot")
.contains(&anchor)
);
}
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
task_id: "fixture-token".to_string(),

Some files were not shown because too many files have changed in this diff Show More