Compare commits

..

4 Commits

Author SHA1 Message Date
唐小鸭 a5e2aab1b4 fix(ecstore): let Windows directory pins share writes on every component
A Windows directory pin shared only reads on ancestors, so a directory
pinned as one object's ancestor refused the write open the kernel takes on
a rename target parent. With nested keys (`a/b` beside `a/b/c`) a parent
object's data-directory publication failed with ERROR_SHARING_VIOLATION
while a child write held the pin. Every pinned component now shares reads
and writes and still excludes delete sharing, which is what pins identity.

Also gate the two grouped dst dir fsync rollback tests to Unix, where the
failure hook exists, and make the rebalance dangling-copy fixture wait for
every drive before it removes copies.
2026-11-04 22:58:10 +08:00
唐小鸭 ae6fa93ac3 fix(ecstore): stop Windows reads from recreating uploads and flush writable
Reading native multipart metadata on Windows took the write-side rename
guard, which opens missing parents with FILE_OPEN_IF. A late completion for
a retired upload therefore recreated the upload directory before reporting
FileNotFound. Pin the existing parent instead, which never creates it.

Windows only flushes a handle opened for writing. The part-integrity proof,
the delete-quorum receipt replay, and the inline rollback backup reopened a
file read-only before syncing it, so every protected part upload failed with
access denied and surfaced as a write quorum error. Open those handles for
writing on Windows.
2026-11-04 12:16:05 +08:00
唐小鸭 c830af66d5 fix(ecstore): reclaim owned multipart dirs and read dirs on Windows
Owned-undo reclaim removed a data directory while its own pin was still
open. A Windows directory pin refuses DELETE sharing, so the rmdir always
failed with a sharing violation and the failed attempt was never reclaimed.
Release the leaf pin right before the non-recursive rmdir; the object pin
still holds the parent identity.

Windows refuses a plain read-only open of a directory, so metadata reads
that meet a directory reported FileAccessDenied where Unix classifies the
directory as an absent file. Reopen an actual directory with backup
semantics so the existing is_dir checks apply on both platforms.

The copy-heal fixture flushed fault bytes through a read-only handle,
which Windows rejects; open it for writing instead.

Cover these paths in the Windows filesystem workflow.
2026-11-03 16:53:47 +08:00
唐小鸭 5345772af4 fix(ecstore): complete and abort native multipart uploads on Windows
Native multipart completion publishes its payload from a TMP staging
parent that differs from the canonical head's parent. Linux pins that
parent with a dedicated payload commit guard, but Windows reused the
metadata guard, so the data-dir rename failed its parent check with
InvalidInput, which surfaced as FileNotFound/NoSuchKey.

- Add a Windows multipart payload commit guard that pins the staging
  tree and reuses the metadata transaction's destination parent handle.
- Run the owned-source rename precondition on Windows before opening the
  exclusive DELETE handle, under a non-exclusive identity anchor, and
  require the prepared handle to name the same file. The precondition's
  own opens previously hit ERROR_SHARING_VIOLATION during abort and
  stale-upload cleanup.
- Recognize ERROR_ACCESS_DENIED on a directory as a directory in the
  remove fallback, so remove/remove_std can delete directories on
  Windows; run each fallback in one blocking dispatch.
- Run the complete/abort and directory-removal tests in the Windows
  filesystem workflow, and add a Windows payload guard regression test.
2026-11-02 17:16:19 +08:00
200 changed files with 2158 additions and 15249 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=16b3cd69887a73b468c84012bf5c0cb7e24360fcaf6ab3c13967020e37ef86d2
sha256-linux=c7c7f0c7ce2610bc0a86a518bfa67a3ed56d862aa44f05b4a92200fabf51d6c7
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+1 -1
View File
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Nightly and manual-dispatch lane owned by
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
-2
View File
@@ -32,8 +32,6 @@ updates:
- "overtrue"
- "majinghe"
ignore:
- dependency-name: "pulsar"
versions: [ "6.9.x" ]
- dependency-name: "object_store"
versions: [ "0.13.x" ]
- dependency-name: "libunftp"
+5 -2
View File
@@ -769,10 +769,12 @@
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"workflow_dispatch",
"schedule"
],
@@ -847,11 +849,12 @@
"workflow": ".github/workflows/e2e-upgrade.yml",
"name": "Upgrade Compatibility",
"roles": [
"pr",
"periodic",
"manual",
"release"
"manual"
],
"events": [
"pull_request",
"push",
"schedule",
"workflow_dispatch"
+1 -60
View File
@@ -113,7 +113,7 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
@@ -497,63 +497,6 @@ jobs:
- name: Check offline enrollment E2E root boundary
run: ./scripts/check_offline_enrollment_e2e.sh
fips-provider:
name: FIPS provider boundaries
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
mode: [default, fips]
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
CARGO_BUILD_JOBS: "2"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-fips-provider
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Install FIPS build tools
if: matrix.mode == 'fips'
run: sudo apt-get install -y cmake golang clang libclang-dev
- name: Verify provider installation
shell: bash
run: |
args=()
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
args+=(--features fips)
fi
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
fi
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
fi
for test_name in "${tests[@]}"; do
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
done
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
# Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1454,7 +1397,6 @@ jobs:
- workspace-shard-evidence
- core-contract
- connect-feature-boundaries
- fips-provider
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
@@ -1482,7 +1424,6 @@ jobs:
needs:
- classify-changes
- connect-feature-boundaries
- fips-provider
- required-checks
- typos
- quick-checks
+28 -2
View File
@@ -17,8 +17,8 @@
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
# and manual dispatches execute the same fail-closed suite.
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".config/e2e-distributed-selection.txt"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
+18 -7
View File
@@ -12,19 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Single-node upgrade and rollback lane.
#
# The weekly Monday schedule, version tags, and workflow_dispatch run this
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
name: Upgrade Compatibility
on:
pull_request:
paths:
- ".github/workflows/e2e-upgrade.yml"
- "crates/e2e_test/src/common.rs"
- "crates/e2e_test/src/distributed/**"
- "crates/e2e_test/src/fake_s3_target/**"
- "crates/e2e_test/src/lib.rs"
- "crates/e2e_test/src/replication_extension_test.rs"
- "crates/e2e_test/src/rpc_fault_proxy.rs"
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/kms/**"
- "crates/storage-api/**"
- "rustfs/**"
- "Cargo.lock"
push:
tags:
- "[0-9]*.[0-9]*.[0-9]*"
schedule:
# 03:17 UTC every Monday.
- cron: "17 3 * * 1"
workflow_dispatch:
@@ -33,7 +44,7 @@ permissions:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
+7 -279
View File
@@ -87,206 +87,18 @@ jobs:
install-build-packaging-tools: 'false'
install-test-tools: 'false'
- name: Download static console assets
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
# Packaging tools are off on this lane, so setup does not install unzip.
if ! command -v unzip >/dev/null 2>&1; then
sudo apt-get update -qq
sudo apt-get install -y -qq unzip
fi
if ! command -v unzip >/dev/null 2>&1; then
echo "unzip is required to extract console assets" >&2
exit 1
fi
mkdir -p ./rustfs/static
verify_sha256() {
local expected="$1"
local file="$2"
local actual=""
local actual_line=""
if command -v sha256sum >/dev/null 2>&1; then
actual_line=$(sha256sum -- "$file") || return 1
elif command -v shasum >/dev/null 2>&1; then
actual_line=$(shasum -a 256 -- "$file") || return 1
else
echo "No SHA-256 verification tool found" >&2
return 1
fi
actual="${actual_line%%[[:space:]]*}"
if [ "$actual" = "$expected" ]; then
echo "SHA256 verified OK: $actual"
return 0
else
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
return 1
fi
}
download_console_assets() {
local curl_bin="curl"
local python_bin="python3"
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
local console_json="console-release.json"
local console_url
local console_sha256
local curl_auth_args=()
console_tag=""
chmod +w ./rustfs/static/LICENSE || true
if ! command -v "$python_bin" >/dev/null 2>&1; then
python_bin="python"
fi
if ! command -v "$python_bin" >/dev/null 2>&1; then
echo "No Python interpreter found for release metadata parsing" >&2
return 1
fi
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
fi
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
import json
import re
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
release = json.load(handle)
tag = release.get("tag_name", "") or "unknown"
for asset in release.get("assets", []):
name = asset.get("name", "")
digest = asset.get("digest", "")
url = asset.get("browser_download_url", "")
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
sha256 = digest.split(":", 1)[1]
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
break
else:
raise SystemExit("no console zip asset with sha256 digest found")
PY
) || return 1
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
echo "Console release metadata is missing URL or SHA-256 digest" >&2
return 1
fi
echo "Console release: ${console_tag}"
echo "Downloading console asset: ${console_url}"
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
verify_sha256 "$console_sha256" console.zip || return 2
unzip -o console.zip -d ./rustfs/static || return 2
}
if download_console_assets; then
rm -f console.zip console-release.json
else
status=$?
rm -f console.zip console-release.json
if [[ "$status" -eq 2 ]]; then
echo "Console asset integrity verification failed" >&2
fi
echo "Failed to download verified console assets" >&2
exit 1
fi
if [[ ! -s ./rustfs/static/index.html ]]; then
echo "Console asset archive is missing static/index.html" >&2
exit 1
fi
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
- name: Build RustFS
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
shell: bash
run: |
# Force rebuild by touching build.rs so a restored cache cannot
# reuse a rustfs artifact compiled before these console assets existed.
touch rustfs/build.rs
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
- name: Verify glibc symbol baseline
run: |
python3 scripts/release/check_glibc.py 2.31 \
target/x86_64-unknown-linux-gnu/release/rustfs
target/x86_64-unknown-linux-gnu/release/rustfs \
target/x86_64-unknown-linux-gnu/release/rustfs-cli
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
- name: Verify console
shell: bash
run: |
set -euo pipefail
find_free_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
data_dir="$(mktemp -d)"
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
server_pid=""
trap '
if [[ -n "${server_pid:-}" ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
fi
rm -rf "$data_dir"
' EXIT
if [[ ! -x "$binary" ]]; then
echo "rustfs binary not found: ${binary}" >&2
exit 1
fi
api_port="$(find_free_port)"
console_port="$(find_free_port)"
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
"$binary" server \
--address "127.0.0.1:${api_port}" \
--console-enable \
--console-address "127.0.0.1:${console_port}" \
--access-key console-smoke \
--secret-key console-smoke-secret \
"$data_dir" >"$log_file" 2>&1 &
server_pid=$!
response=""
for _ in {1..40}; do
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
if [[ "$response" == 200\ text/html* ]]; then
exit 0
fi
sleep 0.25
done
echo "Console endpoint did not return 200 text/html: $response" >&2
cat "$log_file"
exit 1
- name: Build DEB package
id: deb
shell: bash
@@ -534,12 +346,12 @@ jobs:
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
# engineers can download and install the nightly directly. The branch is
# a single-commit orphan rewritten on every build, which keeps the repo
# small while the latest files stay reachable at stable raw URLs.
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
# The dispatch below runs only after this upload and the immutable R2
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
# which still starts from this workflow's workflow_run completion.
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
- name: Publish packages to auto-testing release assets
env:
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -548,90 +360,6 @@ jobs:
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
run: bash scripts/release/publish_nightly_assets.sh
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
# or fine-grained Contents write (the permission GitHub checks for
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
# step, and therefore this job, so the scheduled failure alert fires.
# GitHub allows at most 10 top-level client_payload properties, so the
# schema 2 candidate object is nested under `candidate` and the release
# fields sit beside it.
- name: Dispatch nightly candidate to auto-testing
if: ${{ steps.publish.outputs.candidate_file != '' }}
env:
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
SOURCE_REPO: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
: "${DEB_DATE:?DEB_DATE is required}"
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
--arg event_type "nightly-candidate-ready" \
--arg source_repo "${SOURCE_REPO}" \
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
--arg channel "nightly" \
--arg deb_date "${DEB_DATE}" \
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
'{
event_type: $event_type,
source_repo: $source_repo,
latest_alias_url: $latest_alias_url,
auto_testing_release: $auto_testing_release,
channel: $channel,
deb_date: $deb_date,
nightly_workflow: $nightly_workflow,
candidate: $candidate[0]
}' > "${payload}"
if ! jq -e '
.event_type == "nightly-candidate-ready"
and .source_repo == "rustfs/rustfs"
and .channel == "nightly"
and .deb_date == env.DEB_DATE
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
and .candidate.schema == 2
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
' "${payload}" >/dev/null; then
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
exit 1
fi
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile client_payload "${payload}" \
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-o "${response}" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "Content-Type: application/json" \
--data-binary @"${body}" \
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
exit 1
}
if [[ "${status}" != "204" ]]; then
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
if [[ -s "${response}" ]]; then
cat "${response}" >&2
fi
exit 1
fi
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
+5 -41
View File
@@ -92,17 +92,6 @@ jobs:
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
run: |
set -euo pipefail
# Scheduled runs carry no inputs and the resolver only derives a
# build from a workflow_run payload (which a schedule cannot
# provide), so resolve the newest successful nightly build
# explicitly before invoking it.
if [ -z "${BUILD_RUN_ID:-}" ]; then
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
fi
python3 scripts/resolve_functional_candidate.py
python3 - <<'PY'
import json
@@ -110,15 +99,15 @@ jobs:
from pathlib import Path
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
root.mkdir(parents=True, exist_ok=True)
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = chain["candidate"]["manifest"]
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = candidate["manifest"]
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("testing_sha=" + chain["testing_sha"] + "\n")
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
output.write("package_url=" + manifest["package_url"] + "\n")
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
@@ -132,26 +121,6 @@ jobs:
path: auto-testing
persist-credentials: false
- name: Prepare validation Python runtime
# The pool / ilm / replication validation runners require a managed
# Python, exactly like the chain's validation job provides.
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install validation Python requirements
# Case scripts call python3 and import boto3 at startup. Install the
# functional chain's client into a per-run env so that interpreter
# sees it without changing the runner's shared Python.
run: |
set -euo pipefail
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
test ! -e "${validation_python}"
python3 -m venv "${validation_python}"
"${validation_python}/bin/python" -m pip install \
-r auto-testing/scripts/validation_ilm/requirements.txt
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
- name: Select sweep cases
id: select
env:
@@ -255,8 +224,7 @@ jobs:
# topology, this purge guarantees the next case starts clean even
# when the previous one died mid-run.
for node in ${RUSTFS_NODES}; do
# -n keeps ssh from consuming the case list this loop reads from stdin
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
@@ -275,7 +243,6 @@ jobs:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
run: |
set -euo pipefail
python3 - <<'PY'
@@ -384,10 +351,7 @@ jobs:
${{ runner.temp }}/validation-sweep/sweep-cases.txt
${{ runner.temp }}/validation-sweep/candidate.json
${{ runner.temp }}/validation-sweep/results/
${{ runner.temp }}/validation-sweep/run/
# warn (not error): an early infra failure legitimately leaves no
# per-case evidence, and the failing step is already red.
if-no-files-found: warn
if-no-files-found: error
retention-days: 30
- name: Cleanup environment (after)
+26
View File
@@ -107,6 +107,32 @@ jobs:
shell: pwsh
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
# Native multipart completion publishes its payload from a different
# parent than its metadata, and abort renames under a precondition; both
# need their own Windows handle discipline.
- name: Test native multipart complete and abort publication
shell: pwsh
run: >-
cargo test -p rustfs-ecstore --lib --
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
--nocapture
# Owned-undo reclaim removes a directory it pinned, copy heal persists
# fault bytes through a handle, and metadata reads meet a directory.
- name: Test native multipart reclaim, copy heal, and directory reads
shell: pwsh
run: >-
cargo test -p rustfs-ecstore --lib --
native_multipart_owned_undo
native_multipart_copy_heal
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
--nocapture
- name: Test directory removal fallback
shell: pwsh
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
- name: Test Windows path conversion
shell: pwsh
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
-1
View File
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
### Security
Generated
+195 -377
View File
File diff suppressed because it is too large Load Diff
+17 -14
View File
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-compression = { version = "0.4.51" }
async-trait = "0.1.92"
async-nats = { version = "0.51.0", default-features = false }
axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
memchr = "2.8.3"
minlz = "1.2.3"
reqwest = "0.13.5"
rustfs-kafka-async = { version = "1.4.1" }
rustfs-kafka-async = { version = "1.4.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.54.1" }
tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -212,13 +212,12 @@ serde_urlencoded = "0.7.1"
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
aws-lc-rs = { version = "1.19.0", default-features = false }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.1.0" }
openidconnect = { default-features = false, version = "4.1.0" }
openidconnect = { default-features = false, version = "4.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
@@ -235,7 +234,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "0.2.39" }
jiff = { version = "0.2.38" }
time = { version = "0.3.55" }
# Database
@@ -254,9 +253,9 @@ atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.0" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.129.0" }
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
aws-sdk-sts = { default-features = false, version = "1.124.0" }
aws-sdk-kms = { default-features = false, version = "1.128.0" }
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
aws-sdk-sts = { default-features = false, version = "1.123.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.19.2" }
@@ -338,7 +337,7 @@ strum = { version = "0.28.0" }
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
test-case = "3.4.0"
thiserror = "2.0.21"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
@@ -357,13 +356,13 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.3.0"
zip = "9.1.3"
zstd = "0.14.3"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.7", default-features = false }
dial9 = { version = "0.5.6", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -378,16 +377,16 @@ libunftp = { version = "0.24.0" }
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.1" }
russh = { version = "0.65.0" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.3" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.3", default-features = false }
hotpath = { version = "0.30.2", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
@@ -428,3 +427,7 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
zstd = { path = "crates/zstd-compat" }
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
+3 -3
View File
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
/// Default validation mode is "hop_by_hop".
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
/// Environment variable to enable RFC 7239 "Forwarded" header support.
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
/// RFC 7239 support is enabled by default.
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
/// Environment variable for the maximum allowed proxy hops.
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
-6
View File
@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
/// Queue-store compression is enabled by default to reduce disk footprint.
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
+3 -3
View File
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -12,249 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::{
Client,
error::{ProvideErrorMetadata, SdkError},
primitives::ByteStream,
};
use std::any::TypeId;
use std::collections::BTreeSet;
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
use tokio::time::Instant;
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: Debug + ProvideErrorMetadata + 'static,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
// The workload uses unique unversioned keys. Repeat only explicitly
// advertised lock backpressure; every step still needs its real result.
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().is_some_and(|error| {
let Some(response) = error.raw_response() else {
return false;
};
let Some(service_error) = error.as_service_error() else {
return false;
};
// HEAD has no XML error body, so its SDK service error can
// lack a code even when the server supplies retry headers.
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|| (service_error.code().is_none()
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
response.status().as_u16() == 503
&& namespace_code
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
&& response.headers().get("retry-after") == Some("1")
});
if !retry {
return result;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await;
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
eprintln!(
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
started.elapsed()
);
}
outcome.map_err(|_| {
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
})
}
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
let actual = output.body.collect().await?.into_bytes();
if actual.as_ref() != expected {
return Err(format!(
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
expected.len(),
sha256_hex(expected),
actual.len(),
sha256_hex(&actual)
)
.into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_http_client::test_util::infallible_client_fn;
use aws_smithy_types::body::SdkBody;
use std::sync::atomic::{AtomicUsize, Ordering};
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
let requests = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&requests);
let transport = infallible_client_fn(move |_| {
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
let body = if code.is_empty() {
body.to_string()
} else {
format!("<Error><Code>{code}</Code></Error>")
};
let mut response = http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len());
if let Some(cause) = cause {
response = response.header("x-rustfs-error-cause", cause);
}
if let Some(retry_after) = retry_after {
response = response.header("retry-after", retry_after);
}
response.body(SdkBody::from(body)).expect("scripted S3 response")
});
let config = crate::common::build_test_s3_config(
"http://contention.example.invalid",
"synthetic-access",
"synthetic-secret",
None,
"contention-test",
);
let client = Client::from_conf(config.to_builder().http_client(transport).build());
(s3_probe_client(&client), requests)
}
fn contention() -> Reply {
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
}
#[tokio::test(start_paused = true)]
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt < 3 {
contention()
} else {
(200, "", None, None, "")
}
});
let standard = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
.build(),
);
standard
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
.await
.expect_err("three advertised lock responses exhaust the standard SDK policy");
assert_eq!(requests.load(Ordering::SeqCst), 3);
requests.store(0, Ordering::SeqCst);
let started = Instant::now();
after_lock_contention("PUT recovery", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await??;
assert_eq!(requests.load(Ordering::SeqCst), 4);
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt == 0 {
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
} else {
(200, "", None, None, "")
}
});
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
assert_eq!(requests.load(Ordering::SeqCst), 2);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
for reply in [
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(403, "AccessDenied", None, None, ""),
(404, "NoSuchKey", None, None, ""),
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "ServiceUnavailable", None, Some("1"), ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
] {
let (client, requests) = client(move |_| reply);
let result = after_lock_contention("PUT fatal response", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await?;
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn persistent_lock_contention_exhausts_the_request_budget() {
let (client, requests) = client(|_| contention());
let started = Instant::now();
let result = after_lock_contention("PUT unavailable", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await;
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
assert!(requests.load(Ordering::SeqCst) > 1);
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
}
#[tokio::test(start_paused = true)]
async fn acknowledged_wrong_bytes_fail_without_retry() {
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
.await
.expect_err("a successful but incorrect read is a consistency failure");
assert!(error.to_string().contains("bytes mismatch"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
// This consistency probe owns its admission policy; unrelated responses
// must not be hidden by the SDK's general-purpose retries.
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
let context = format!("worker={idx} round={round}");
after_lock_contention(&format!("{context} PUT source"), || {
writer
.put_object()
.bucket(&bucket)
.key(&key)
.body(ByteStream::from(body.clone()))
.send()
})
.await??;
put_object(writer, &bucket, &key, body.clone()).await?;
let head = after_lock_contention(&format!("{context} HEAD source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await??;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_workload_bytes(reader, &bucket, &key, &body).await?;
assert_object_bytes(reader, &bucket, &key, &body).await?;
after_lock_contention(&format!("{context} COPY retained"), || {
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
})
.await??;
assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
after_lock_contention(&format!("{context} DELETE source"), || {
writer.delete_object().bucket(&bucket).key(&key).send()
})
.await??;
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await?
.expect_err("deleted source key must not remain visible");
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
for handle in handles {
inventory.extend(handle.await??);
}
assert_eq!(
inventory.len(),
worker_count * rounds,
"every worker round must retain its acknowledged copy"
);
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
})
.await??;
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_workload_bytes(client, &bucket, key, body)
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
@@ -33,7 +33,7 @@ mod tests {
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::error::Error;
use std::io::{Read, Seek, SeekFrom, Write};
use std::io::{Read, Write};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::process::Command;
@@ -1654,89 +1654,6 @@ mod tests {
TargetEndpointBlackhole,
}
async fn release_heal_commit_barrier_after_shutdown(
log: &Path,
log_offset: u64,
barrier: &Path,
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
// Observe within the existing graceful process-stop budget.
timeout(Duration::from_secs(45), async {
loop {
let mut file = std::fs::File::open(log)?;
file.seek(SeekFrom::Start(log_offset))?;
let mut tail = String::new();
file.read_to_string(&mut tail)?;
if let Some(event) = tail
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.find(|event| {
event["event"] == "server_shutdown_state"
&& event["component"] == "main"
&& event["subsystem"] == "startup"
&& event["state"] == "stopping"
})
{
// Local heal has stopped and HTTP shutdown has been signalled.
// Let admitted peer RPCs finish; keeping this synthetic barrier
// until process exit would make the fixture block its own drain.
std::fs::remove_file(barrier)?;
return Ok(event);
}
sleep(Duration::from_millis(10)).await;
}
})
.await?
}
#[tokio::test(start_paused = true)]
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
let directory = tempfile::tempdir()?;
let log = directory.path().join("server.log");
let barrier = directory.path().join("barrier");
let event = serde_json::json!({
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
});
std::fs::write(&log, format!("{event}\n"))?;
std::fs::write(&barrier, "bucket/prefix/")?;
let log_offset = log.metadata()?.len();
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
tokio::pin!(release);
assert!(
futures::poll!(release.as_mut()).is_pending(),
"old or unrelated events must not release the barrier"
);
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
tokio::time::advance(Duration::from_secs(6)).await;
assert!(
futures::poll!(release.as_mut()).is_pending(),
"background shutdown may take longer than five seconds within the process-stop budget"
);
writeln!(writer, "{event}")?;
tokio::time::advance(Duration::from_millis(10)).await;
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
let directory = tempfile::tempdir()?;
let log = directory.path().join("server.log");
let barrier = directory.path().join("barrier");
std::fs::write(&log, "")?;
std::fs::write(&barrier, "bucket/prefix/")?;
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
.await
.expect_err("missing shutdown evidence must fail the fixture");
assert!(error.is::<tokio::time::error::Elapsed>());
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
Ok(())
}
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
let server_binary = rustfs_binary_path();
let evidence_case = match scenario {
@@ -1874,7 +1791,6 @@ mod tests {
let mut created_online_objects = Vec::with_capacity(online_object_count);
let mut expected_manifests = Vec::with_capacity(online_object_count);
let mut unclean_shutdown_marker_observed = None;
let mut shutdown_barrier_release = None;
let mut attempt_count = 0usize;
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
while expected_manifests.len() < online_object_count {
@@ -2457,21 +2373,7 @@ mod tests {
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
| InterruptionScenario::BackgroundCoordinatorRestart
) {
if interruption_node == 1
&& let Some(barrier) = &commit_barrier
{
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
let log_offset = target_log.metadata()?.len();
let (stopped, released) = tokio::join!(
cluster.stop_node_gracefully(interruption_node),
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
);
// Await both branches even on error so the child is reaped.
stopped?;
shutdown_barrier_release = Some(released?);
} else {
cluster.stop_node_gracefully(interruption_node).await?;
}
cluster.stop_node_gracefully(interruption_node).await?;
} else {
cluster.stop_node(interruption_node)?;
}
@@ -2513,9 +2415,7 @@ mod tests {
}
}
}
if shutdown_barrier_release.is_none()
&& let Some(barrier) = &commit_barrier
{
if let Some(barrier) = &commit_barrier {
std::fs::remove_file(barrier)?;
}
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2846,7 +2746,6 @@ mod tests {
"all_peers_bound_to_generation_window": true,
"pid_before": target_pid, "pid_after": restarted_pid,
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
"graceful_barrier_release": shutdown_barrier_release,
"objects": evidence_objects, "node_listings": node_listings,
});
if let Some(outage_write) = outage_write_diagnostic {
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
ServerSideEncryption, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
VersioningConfiguration,
};
use base64_simd::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
init_logging();
let mut source_env = RustFSTestEnvironment::new().await?;
let source_log = format!("{}/source.log", source_env.temp_dir);
source_env.capture_log_path = Some(source_log.clone());
let mut source_env_vars = replication_fast_env();
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
"target did not preserve the source delete-marker version ID"
);
let wait_for_source_writeback = || async {
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
timeout(Duration::from_secs(30), async {
loop {
let response = signed_request(
http::Method::GET,
&metrics_url,
&source_env.access_key,
&source_env.secret_key,
None,
None,
)
.await?;
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
let queued = metrics["queued"]["curr"]["count"]
.as_f64()
.ok_or("missing replication queue count")?;
if metrics["provider_available"] == true && queued == 0.0 {
return Ok::<(), Box<dyn Error + Send + Sync>>(());
}
sleep(Duration::from_millis(50)).await;
}
})
.await?
};
wait_for_source_writeback().await?;
let purged = source_client
.delete_objects()
source_client
.delete_object()
.bucket(source_bucket)
.delete(
Delete::builder()
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(&purged_version_id)
.build()?,
)
.objects(
ObjectIdentifier::builder()
.key(object_key)
.version_id(delete_marker_version_id)
.build()?,
)
.build()?,
)
.key(object_key)
.version_id(&purged_version_id)
.send()
.await?;
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
let target_state = list_replication_state(&target_client, target_bucket).await?;
assert!(
target_state.iter().all(|entry| entry.version_id != purged_version_id),
"target retained the explicitly purged object version"
);
assert!(
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
"target retained the explicitly purged delete marker"
);
assert!(
target_state
.iter()
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
.await?;
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
// Wait for the source workers to finish their final writeback, which runs
// after the remote purge has already become visible (rustfs#9121).
wait_for_source_writeback().await?;
let logs = fs::read_to_string(source_log).await?;
let writeback_failures: Vec<_> = logs
.lines()
.filter(|line| line.contains("apply_replication_delete_state"))
.collect();
assert!(
writeback_failures.is_empty(),
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
);
Ok(())
}
-1
View File
@@ -31,7 +31,6 @@ workspace = true
[features]
default = []
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
# Sampled wall/CPU diagnostics only; does not change the read backend.
diagnostic-local-batch = ["rustix/time"]
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
impl ReplicationMetadataStore {
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
@@ -117,10 +116,6 @@ mod tests {
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
"config/replication/mrf.bin.recovery"
);
assert_eq!(
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
"config/replication/mrf.bin.replay"
);
assert_eq!(
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
"config/replication/force-delete.bin"
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
const MRF_PENDING_CAP: usize = 200_000;
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
/// re-attempts every retained entry, so a target that stays down costs one
/// attempt per entry per interval.
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
#[derive(Debug, Default)]
pub struct DurableMrfBacklog {
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
}
/// Acknowledge only the generation read by the recovery leader. Entries appended
/// after that generation are retained as a suffix and replayed on the next pass.
/// after that generation are retained as a suffix and replayed on the next startup.
/// Lock order: recovery leader lock -> MRF journal object lock.
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
storage: Arc<S>,
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
Err(EcstoreError::PreconditionFailed)
}
/// Acquires the MRF recovery leader lock for one journal replay pass.
/// Acquires the MRF recovery leader lock for the startup replay.
/// Returns `None` (after logging) when the lock cannot be created or another
/// node is already processing the backlog.
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
}
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
/// On acknowledgement failure the backlog is preserved for the next replay pass and
/// On acknowledgement failure the backlog is preserved for the next startup and
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
storage: &Arc<S>,
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
}
}
fn unix_millis_now() -> i64 {
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
}
/// Whether a periodic pass may run, given the start time of the last pass on
/// any node. A stamp more than one interval ahead of this node's clock is skew
/// or a bad value, so it does not suppress replay.
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
let Some(last_started_ms) = last_started_ms else {
return true;
};
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
}
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
.await
.ok()?;
std::str::from_utf8(&data).ok()?.trim().parse().ok()
}
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
if let Err(error) = ReplicationConfigStore::save(
storage.clone(),
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
started_ms.to_string().into_bytes(),
)
.await
{
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to record the MRF replay start time"
);
}
}
/// One MRF journal replay pass. The recovery leader reads the journal,
/// re-drives every entry, and acknowledges the replayed prefix; entries that
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
///
/// Every node runs the periodic loop, and the leader lock only serializes
/// passes. With `min_interval`, a pass is skipped when a pass on any node
/// started within that interval, read from the shared replay stamp, so a
/// cluster replays once per interval rather than once per node. The startup
/// pass passes `None` and always runs. A pass that finds entries records its
/// start time. Returns whether the journal was read.
/// Lock order: recovery leader lock -> replay stamp object lock.
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
return false;
};
let started_ms = unix_millis_now();
if let Some(interval) = min_interval
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
{
return false;
}
let Some(entries) = load_mrf_recovery_entries(storage).await else {
return true;
};
if !entries.is_empty() {
write_mrf_replay_stamp(storage, started_ms).await;
}
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
true
}
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
/// one pass at a time across the cluster, including the startup pass, and the
/// shared replay stamp spaces passes from different nodes by `period`.
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
tokio::spawn(async move {
loop {
tokio::time::sleep(period).await;
replay_mrf_journal(&storage, Some(period)).await;
}
})
}
#[derive(Debug, thiserror::Error)]
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
struct ResyncActiveConflictError {
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// Start background tasks
pool.start_mrf_persister().await;
pool.start_mrf_processor().await;
pool.start_mrf_replayer().await;
pool.start_force_delete_processor().await;
pool
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Starts the MRF processor — one-shot at startup.
///
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
/// the pass while the server runs.
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
/// rewrites any entries that could not be admitted for a later startup retry.
async fn start_mrf_processor(&self) {
let storage = self.storage.clone();
let handle = tokio::spawn(async move {
replay_mrf_journal(&storage, None).await;
});
self.task_handles.lock().await.push(handle);
}
/// Starts the runtime MRF replayer — ongoing background task.
///
/// Entries reach the journal while the server runs (worker-queue overflow,
/// delete-marker purge intents whose watch window drained with targets still
/// dirty). Replaying only at startup left them waiting for a restart.
async fn start_mrf_replayer(&self) {
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
let handle = tokio::spawn(async move {
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
return;
};
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
return;
};
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
retry_entries.push(entry.clone());
} else if admission == ReplicationQueueAdmission::Queued {
queued_count += 1;
}
}
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
if queued_count > 0 {
info!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
recovered = queued_count,
total,
retained = retained_count,
"Replayed MRF entries admitted for retry"
);
}
});
self.task_handles.lock().await.push(handle);
}
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// writes them to the on-disk MRF file every flush interval (default 10s,
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
/// entries accumulate. Runtime writers append under the journal lock; only the
/// recovery leader of a replay pass may remove an acknowledged prefix.
/// startup recovery leader may remove an acknowledged prefix.
async fn start_mrf_persister(&self) {
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
return;
@@ -3511,8 +3421,6 @@ mod tests {
allow_first_read: Notify,
read_count: AtomicUsize,
delete_storage_calls: AtomicUsize,
delete_error: StdMutex<EcstoreError>,
replay_stamp: StdMutex<Option<Vec<u8>>>,
reported_size: StdMutex<Option<i64>>,
stream_read_bytes: Arc<AtomicUsize>,
write_count: AtomicUsize,
@@ -3580,24 +3488,6 @@ mod tests {
_h: Self::HeaderMap,
_opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader, Self::Error> {
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let stamp = self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned")
.clone()
.ok_or(EcstoreError::FileNotFound)?;
return Ok(Self::GetObjectReader {
object_info: ObjectInfo {
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
..Default::default()
},
stream: Box::new(Cursor::new(stamp)),
buffered_body: None,
body_source: Default::default(),
});
}
if !object.ends_with("/.replication/resync.bin")
&& !object.ends_with("config/replication/mrf.bin")
&& !object.ends_with("config/replication/force-delete.bin")
@@ -3661,16 +3551,6 @@ mod tests {
let lock = self.new_ns_lock(bucket, object).await?;
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
};
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
let mut stamp = Vec::new();
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
*self
.shared
.replay_stamp
.lock()
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
return Ok(ObjectInfo::default());
}
if opts.http_preconditions.is_some()
&& let Some(replacement) = self
.shared
@@ -3792,12 +3672,7 @@ mod tests {
_opts: Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error> {
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
Err(self
.shared
.delete_error
.lock()
.expect("delete error lock should not be poisoned")
.clone())
Err(EcstoreError::NotImplemented)
}
async fn delete_objects(
@@ -4296,8 +4171,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -4348,103 +4221,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_worker_already_purged_source_completes_without_retry() {
let arn = "arn:test";
for delete_marker_version in [false, true] {
for version_id in [Uuid::new_v4(), Uuid::nil()] {
for error in [
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
EcstoreError::FileVersionNotFound,
EcstoreError::FileNotFound,
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: (!delete_marker_version).then_some(version_id),
delete_marker_version_id: delete_marker_version.then_some(version_id),
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
replicate_delete_with_outcome(delete, storage).await,
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
);
assert_eq!(
shared.delete_storage_calls.load(Ordering::SeqCst),
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
"the worker must lock and attempt writeback, reading a non-null data version first"
);
}
}
}
}
#[tokio::test]
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
let arn = "arn:test";
for (version_purge, target_status, error) in [
(
true,
VersionPurgeStatusType::Pending,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
(
false,
VersionPurgeStatusType::Empty,
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
),
] {
let shared = empty_resync_shared_state();
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
let delete = DeletedObjectReplicationInfo {
bucket: "source".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "purged".to_string(),
version_id: version_purge.then(Uuid::new_v4),
delete_marker: !version_purge,
replication_state: Some(ReplicationState {
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
targets: if version_purge {
HashMap::new()
} else {
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
},
purge_targets: if version_purge {
HashMap::from([(arn.to_string(), target_status)])
} else {
HashMap::new()
},
..Default::default()
}),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
};
assert!(
!replicate_delete_with_outcome(delete, storage).await,
"unfinished target work, source I/O failures, and marker creation must remain retryable"
);
}
}
#[tokio::test]
async fn delete_worker_preserves_persisted_admission_without_live_config() {
let arn = "arn:test";
@@ -5528,8 +5304,6 @@ mod tests {
allow_first_read: Notify::new(),
read_count: AtomicUsize::new(0),
delete_storage_calls: AtomicUsize::new(0),
delete_error: StdMutex::new(EcstoreError::NotImplemented),
replay_stamp: StdMutex::new(None),
reported_size: StdMutex::new(None),
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
write_count: AtomicUsize::new(0),
@@ -6494,108 +6268,6 @@ mod tests {
assert_eq!(retained.target_arns, entry.target_arns);
}
#[tokio::test]
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
let shared = empty_resync_shared_state();
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
// A force-delete intent from a superseded generation replays as
// Skipped, so a pass that reaches it acknowledges it off the journal.
let stale_intent = |object: &str| MrfReplicateEntry {
bucket: "mrf-runtime-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
let journal_drained = || async {
tokio::time::timeout(Duration::from_secs(5), async {
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.is_ok()
};
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
for object in ["first", "second"] {
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
assert!(
journal_drained().await,
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
);
}
handle.abort();
}
#[tokio::test]
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
let shared = empty_resync_shared_state();
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
let interval = Duration::from_secs(1);
let persist_backlog = |object: &str| {
let entry = MrfReplicateEntry {
bucket: "mrf-node-replay".to_string(),
object: object.to_string(),
op: MrfOpKind::Delete,
force_delete_id: Some(uuid::Uuid::new_v4()),
force_delete_local_commit: false,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
..Default::default()
};
*shared.data.lock().expect("test data lock should not be poisoned") =
encode_mrf_file(&[entry]).expect("MRF entry should encode");
};
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
persist_backlog("first");
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
// Node B's timer fires within the same interval with a backlog present.
persist_backlog("second");
assert!(
!replay_mrf_journal(&node_b, Some(interval)).await,
"a node must not replay within one interval of another node's pass"
);
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
assert!(
replay_mrf_journal(&node_b, None).await,
"the startup pass should replay regardless of the shared stamp"
);
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
persist_backlog("third");
tokio::time::sleep(interval).await;
assert!(
replay_mrf_journal(&node_a, Some(interval)).await,
"a periodic pass should replay once the interval has elapsed"
);
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
}
#[test]
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
let interval = Duration::from_secs(300);
let now = 1_000_000_000;
assert!(mrf_replay_due(None, now, interval));
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
assert!(
!mrf_replay_due(Some(now + 1_000), now, interval),
"small clock skew should still space passes"
);
assert!(
mrf_replay_due(Some(now + 300_000), now, interval),
"a stamp an interval ahead must not suppress replay"
);
}
#[test]
fn mrf_entry_delete_marker_roundtrip() {
let dm_vid = Uuid::new_v4();
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
true
}
Err(e) => {
// A batch purge or another worker may have removed this version
// already. Remote failures still control the final outcome below.
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
if !source_already_purged {
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
}
error!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %dobj.target_arn,
object = %dobj.delete_object.object_name,
operation = "apply_replication_delete_state",
error = %e,
"Replication target operation failed"
);
send_event(EventArgs {
event_name,
bucket_name: bucket.clone(),
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
},
..Default::default()
});
source_already_purged
false
}
};
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
/// If the marker disappears (deleted before or while the replica landed),
/// purge the replicated marker from the targets, retrying failed targets on
/// later rounds. When the window drains with targets still dirty, persist the
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
/// purge intent as a durable MRF entry so the next startup replays it through
/// `purge_stale_delete_marker_targets`.
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
bucket: String,
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
/// Heal callers retry for the full watch window because the startup MRF
/// processor runs before bucket metadata (and thus target clients) finishes
/// initializing — the first pass can see `target_client_missing` and a later
/// round resolves the client; replay passes are serial background work, so
/// round resolves the client; the replay loop is serial and startup-only, so
/// blocking it for up to the window per dirty entry is acceptable. Live
/// callers run on replication workers where a down target would pin a worker
/// for the whole window, so they attempt once and lean on the durable intent
+3 -1
View File
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
};
use hmac::{Hmac, KeyInit, Mac};
use http::uri::Authority;
use http::{HeaderMap, HeaderValue, Method, Uri};
#[cfg(test)]
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
global_internode_metrics,
};
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
use rustfs_utils::get_env_bool;
use sha2::Digest as _;
use sha2::Sha256;
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
use tracing::{error, info, warn};
use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -1637,7 +1637,6 @@ impl RemoteDisk {
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
if initial_probe_ok {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1709,7 +1708,6 @@ impl RemoteDisk {
// Perform basic connectivity check
if Self::perform_connectivity_check(&addr).await.is_ok() {
health.record_operation_success(&endpoint, "connectivity_probe_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
warn!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -1780,7 +1778,6 @@ impl RemoteDisk {
}
_ = interval.tick() => {
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
debug!(
event = EVENT_REMOTE_DISK_HEALTH,
@@ -2012,7 +2009,6 @@ impl RemoteDisk {
let operation_result = operation().await;
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -2027,7 +2023,6 @@ impl RemoteDisk {
// Log success; the waiting guard balances every exit path.
if operation_result.is_ok() {
self.health.record_operation_success(&self.endpoint, "operation_success");
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
.await;
@@ -5745,7 +5740,6 @@ mod tests {
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
format_data: Bytes,
read_all_data: Bytes,
fail_format_reads: Arc<AtomicBool>,
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
unsupported_methods: Arc<StdMutex<Vec<String>>>,
}
@@ -5765,7 +5759,6 @@ mod tests {
object_read_all_disks: Arc::default(),
format_data,
read_all_data,
fail_format_reads: Arc::default(),
next_format_read_gate: Arc::default(),
unsupported_methods: Arc::default(),
}
@@ -5889,9 +5882,6 @@ mod tests {
gate.entered.notify_one();
gate.release.cancelled().await;
}
if peer.fail_format_reads.load(Ordering::Acquire) {
return Err(Status::unavailable("format read temporarily unavailable"));
}
peer.format_data.clone()
} else {
peer.object_read_all_disks
@@ -8857,124 +8847,6 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
temp_env::async_with_vars(
[
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
(rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
],
async {
runtime_sources::ensure_test_rpc_secret();
let mut reference = crate::layout::format::FormatV3::new(1, 1);
reference.erasure.this = reference.erasure.sets[0][0];
let peer = TestGrpcPeer::spawn(
Bytes::from(reference.to_json().expect("format should serialize")),
Bytes::from_static(b"recovered-peer-data"),
)
.await
.expect("authenticated regression peer is required");
let set = missing_read_set(&peer, reference).await;
let disk = crate::disk::new_disk(
&set.set_endpoints[0],
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.expect("registered remote disk should construct");
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
panic!("fixture must create a remote disk");
};
assert!(remote.health_check);
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
set.disks.write().await[0] = Some(Arc::clone(&disk));
peer.peer.fail_format_reads.store(true, Ordering::Release);
set.connect_disks().await;
peer.peer.fail_format_reads.store(false, Ordering::Release);
let cancelled_after_failure = remote.cancel_token.is_cancelled();
assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
assert!(
Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
"a failed reconnect must not publish its unvalidated candidate"
);
remote.spawn_recovery_monitor_if_needed();
let recovery = time::timeout(Duration::from_secs(5), async {
while !disk.is_online().await {
tokio::task::yield_now().await;
}
disk.read_all("bucket", "object").await
})
.await;
disk.close().await.expect("stop the registered disk's monitoring");
peer.stop().await;
assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
assert_eq!(
recovery
.expect("failed reconnect must leave the registered disk able to recover without another sweep")
.expect("the recovered registered disk must serve real I/O"),
Bytes::from_static(b"recovered-peer-data")
);
},
)
.await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
runtime_sources::ensure_test_rpc_secret();
let mut reference = crate::layout::format::FormatV3::new(1, 1);
reference.erasure.this = reference.erasure.sets[0][0];
let peer = TestGrpcPeer::spawn(
Bytes::from(reference.to_json().expect("format should serialize")),
Bytes::from_static(b"replacement-peer-data"),
)
.await
.expect("authenticated regression peer is required");
let set = missing_read_set(&peer, reference).await;
let disk = crate::disk::new_disk(
&set.set_endpoints[0],
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.expect("registered remote disk should construct");
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
panic!("fixture must create a remote disk");
};
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
set.disks.write().await[0] = Some(Arc::clone(&disk));
let gate = peer.peer.pause_next_format_read();
let pending_set = Arc::clone(&set);
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
time::timeout(Duration::from_secs(5), gate.entered.notified())
.await
.expect("replacement must reach the format validation barrier");
let cancelled_before_publication = remote.cancel_token.is_cancelled();
gate.release.cancel();
pending.await.expect("validated replacement must finish publication");
let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
let readback = replacement.read_all("bucket", "object").await;
replacement.close().await.expect("stop replacement monitoring");
peer.stop().await;
assert!(
!cancelled_before_publication,
"the registered monitor must survive pending format validation"
);
assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
assert_eq!(
readback.expect("replacement must serve real I/O"),
Bytes::from_static(b"replacement-peer-data")
);
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
@@ -445,10 +445,6 @@ impl RemoteClient {
}
}
});
// A successful lock RPC over a cached or fresh channel proves the peer is
// reachable; reconcile the internode peer-health gauge so it does not stay
// stale when no fresh dial fires (issue #9119).
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
fn admit_late_release(&self) -> bool {
@@ -676,7 +672,6 @@ impl RemoteClient {
with_lock_peer_health(&addr, |health| {
health.last_success = Some(Instant::now());
});
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
if let Some(late) = late {
late(response).await;
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
use std::path::PathBuf;
use tokio::fs;
use uuid::Uuid;
struct PhysicalFixture {
_directories: Vec<tempfile::TempDir>,
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
);
}
}
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
let fixture = fixture(0).await;
for pool in disks(&fixture.store).await {
for disk in pool {
let object = object_path(&disk);
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
if entry.file_type().await.expect("fixture entry type").is_dir() {
fs::remove_dir_all(entry.path())
.await
.expect("remove task-owned fixture rollback directory");
}
}
}
}
fixture
}
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
let mut all = Vec::new();
for pool in &store.pools {
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
let members = set
.disks
.read()
.await
.iter()
.cloned()
.map(|disk| disk.expect("identity fixture disk must be online"))
.collect::<Vec<_>>();
let mut heads = Vec::new();
for disk in members {
match disk
.pool_meta_native_recovery(Request::InspectIdentity {})
.await
.expect("physical identity inspection")
{
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
_ => panic!("identity inspection must return its physical head"),
}
}
all.push(heads);
}
all
}
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
let fixture = owner_repair_no_backups_fixture().await;
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("fixture must provide valid committed pool metadata"),
};
meta.version = version;
let raw = meta
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
.expect("legacy fixture should encode in its original version");
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
let all = disks(&fixture.store).await;
let now = OffsetDateTime::now_utc();
for pool in &all {
let heads = replicas(&raw, now).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install exact legacy head");
}
}
if stale {
let old = replicas(&raw, now - Duration::seconds(10)).await;
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
.await
.expect("install stale same-raw legacy head");
}
(fixture, raw)
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
drop(state);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for read in &reads {
assert!(
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
let mut source_raw = raw.clone();
// A second MessagePack value is readable by legacy decoding but
// must not be copied as part of an unverified repair payload.
source_raw.push(0xc0);
assert!(
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
"ordinary legacy reads must remain compatible with trailing bytes"
);
let all_disks = disks(&fixture.store).await;
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in all_disks[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install readable legacy source with trailing bytes");
}
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await
.expect("legacy semantic selection must remain readable");
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("legacy native heal must terminate");
drop(state);
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"version {version}: no identity shard may change"
);
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
for target_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
_ => panic!("initial V1 metadata must decode"),
};
meta.version = POOL_META_VERSION;
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V1 payload must remain valid"),
};
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
PoolMetaReplica::Valid { canonical, .. } => canonical,
_ => panic!("V2 payload must remain valid"),
};
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
// Pool two follows pool zero; pools zero and one cover both
// orderings and both target formats in separate runs.
let versions = [first_version, other_version, first_version];
let physical = disks(&fixture.store).await;
let timestamp = OffsetDateTime::now_utc();
for (pool_index, pool) in physical.iter().enumerate() {
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
let heads = replicas(raw, timestamp).await;
for (index, disk) in pool.iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
}
}
let before = snapshots(&fixture.store).await;
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
.await.expect("equivalent legacy pools should be selectable");
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
tokio::time::timeout(std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
.await.expect("mixed legacy heal must terminate")
.expect("equivalent legacy metadata should heal without downgrading the format floor");
drop(state);
let after = snapshots(&fixture.store).await;
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
for (pool_index, read) in reads.iter().enumerate() {
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
assert!(matches!(&read.replica,
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
for (index, snapshot) in after[pool_index].iter().enumerate() {
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
.await.expect("every physical shard must match its selected legacy payload");
}
if pool_index != target_pool {
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
}
}
}
}
},
).await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
for prepare_pool in [0, 1] {
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
PoolMetaReplica::Valid {
meta,
canonical,
revision,
committed: true,
..
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
_ => panic!("baseline V1 metadata must be committed and readable"),
};
let previous = PoolMetaCommittedCandidate {
canonical: legacy_canonical,
meta: legacy_meta.clone(),
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
};
let mut next = legacy_meta;
next.version = POOL_META_GENERATION_VERSION;
next.pools[0].last_update += Duration::seconds(1);
let revision = PoolMetaRevision {
version: POOL_META_GENERATION_VERSION,
cluster_id: fixture.revision.cluster_id,
epoch: fixture.revision.epoch,
generation: 1,
transaction_id: Some(Uuid::new_v4()),
};
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
.expect("V3 PREPARE with V1 predecessor must encode");
assert!(matches!(decode_pool_meta_replica(pending.clone()),
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
let physical = disks(&fixture.store).await;
for (index, disk) in physical[prepare_pool].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &heads[index])
.await
.expect("install task-owned V3 PREPARE evidence");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[prepare_pool].replica,
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
if raw == &pending && *actual == revision));
let other_pool = 1 - prepare_pool;
assert!(matches!(&reads[other_pool].replica,
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture
.store
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
)
.await
.expect("mixed V1/V3 PREPARE heal must terminate");
drop(state);
assert!(
result.is_err(),
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"pool {prepare_pool}: rejection must not mutate identity evidence"
);
}
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("current reader must recognize the legacy source"),
};
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source && revision.version == POOL_META_V1_VERSION));
let physical = disks(&fixture.store).await;
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[0].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
.await
.expect("install legacy source on pool zero");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve every original head and backup"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
"the dangerous source bytes must not be copied into the target pool"
);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(pool_meta_version_env)]
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
],
async {
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
meta, committed: true, ..
} => meta,
_ => panic!("baseline V1 metadata must decode"),
};
let statuses = meta
.pools
.iter()
.map(|pool| {
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
LegacyPoolStatus {
id: pool.id,
cmd_line: pool.cmd_line.clone(),
last_update: pool.last_update,
decommission: None,
}
})
.collect();
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: statuses,
dont_save: true,
};
let mut dangerous_raw = Vec::new();
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
let old_reader: LegacyPoolMeta =
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
PoolMetaReplica::Valid {
canonical,
revision,
committed: true,
..
} => {
assert_eq!(revision.version, POOL_META_V1_VERSION);
canonical
}
_ => panic!("safe pool-zero source must decode"),
};
assert!(
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
"the dangerous target must remain semantically equivalent to the safe source"
);
let physical = disks(&fixture.store).await;
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
for (index, disk) in physical[1].iter().enumerate() {
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
.await
.expect("install legacy dangerous target head");
}
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
)
.await
.expect("unsafe legacy target heal must terminate");
drop(state);
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
assert!(
error.to_string().contains("dont_save"),
"rejection must identify the unsafe legacy field: {error}"
);
assert_eq!(
snapshots(&fixture.store).await,
before,
"rejection must preserve all physical heads and backups"
);
assert_eq!(
owner_repair_identity_heads(&fixture.store).await,
identities,
"rejection must not write identity shards"
);
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
assert!(
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
"unsafe target evidence must not be overwritten by the safe source"
);
},
)
.await;
}
+6 -281
View File
@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
#[cfg(test)]
use std::io::Cursor;
use std::io::Write;
use std::path::PathBuf;
@@ -6838,47 +6839,11 @@ impl Drop for PoolMetaTransactionArm {
}
}
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
if data.len() >= 4
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
// The selection already validates the payload. The runtime-only
// flag can only appear in the legacy three-field sequence or map.
&& rmp::decode::read_array_len(&mut &data[4..])
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
.ok()
== Some(3)
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
&& legacy.dont_save
{
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
}
Ok(())
}
// Ordinary legacy reads retain their compatibility behavior, but bytes that
// will be copied to another pool must not contain an unparsed extension.
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
let payload = data
.get(4..)
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
if decoder.get_ref().position() != payload_len {
return Err(Error::other("pool metadata legacy source has trailing bytes"));
}
Ok(())
}
#[derive(Debug)]
struct PoolMetaSelection {
meta: PoolMeta,
revision: PoolMetaRevision,
canonical: Option<Vec<u8>>,
// Legacy semantic comparison canonicalizes to V2; a heal must instead
// preserve verified raw bytes at the highest observed format floor.
legacy_raw: Option<Vec<u8>>,
legacy_has_unsafe_flag: bool,
replica_state: PoolMetaReplicaState,
cas_tokens: Vec<PoolMetaCasToken>,
absent: bool,
@@ -7269,8 +7234,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: PoolMeta::default(),
revision: PoolMetaRevision::legacy(0),
canonical: None,
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair: false,
repair_write_safe: true,
@@ -7325,8 +7288,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
meta: selected.meta.clone(),
revision: selected.revision,
canonical: Some(selected.canonical.clone()),
legacy_raw: None,
legacy_has_unsafe_flag: false,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -7340,10 +7301,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
// record; divergent backups without it are ambiguous and fail closed.
let legacy_has_unsafe_flag = committed
.iter()
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
let mut floor_raw = None;
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
for (idx, raw, candidate) in committed {
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7357,36 +7314,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
}
} else {
needs_repair |= selected_raw != &raw;
if floor_raw.is_none()
&& candidate.revision.version == observed_version
&& raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(raw);
}
}
} else {
selected = Some((idx, raw, candidate));
}
}
let (_, selected_raw, mut selected) =
let (_, _, mut selected) =
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
if selected.revision.version == observed_version
&& selected_raw
.get(2..4)
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
{
floor_raw = Some(selected_raw);
}
selected.meta.version = observed_version;
selected.revision.version = observed_version;
Ok(PoolMetaSelection {
meta: selected.meta,
revision: selected.revision,
canonical: Some(selected.canonical),
legacy_raw: floor_raw,
legacy_has_unsafe_flag,
replica_state: PoolMetaReplicaState {
needs_repair,
repair_write_safe,
@@ -8380,9 +8320,8 @@ where
})
});
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
if matches!(&result, Err(Error::PreconditionFailed)) {
let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
if !may_have_mutated {
record_pool_meta_stale_write_rejection(phase);
}
let mut source = None;
@@ -8393,8 +8332,6 @@ where
source = Some(Arc::clone(&original));
if matches!(original.as_ref(), Error::PreconditionFailed) {
Err(Error::PreconditionFailed)
} else if write_not_started {
Err(Error::other(original))
} else {
Err(Error::other(pool_metadata_error(
crate::error::PoolMetadataFailure::TransactionUnknown,
@@ -14609,23 +14546,6 @@ impl ECStore {
.canonical
.as_ref()
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
}
if selection.legacy_has_unsafe_flag {
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
}
let repair_data = if selection.revision.is_generation_protocol() {
canonical.clone()
} else {
let raw = selection
.legacy_raw
.clone()
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
ensure_legacy_pool_meta_payload_complete(&raw)?;
raw
};
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
let pool = self
.pools
.get(pool_index)
@@ -14636,10 +14556,10 @@ impl ECStore {
.get(pool_index)
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
let mut arm = state.arm_transaction();
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
.await?;
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
if confirmed != repair_data {
if confirmed != *canonical {
return Err(Error::other("pool metadata native heal readback diverged"));
}
fence.ensure_held()?;
@@ -29562,53 +29482,6 @@ mod tests {
);
}
#[test]
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
let frame = |payload: Vec<u8>| {
let mut data = Vec::new();
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
data.extend_from_slice(&payload);
data
};
let two_fields = PersistedPoolMetaV1 {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
};
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
let data = frame(payload.expect("two-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
for dont_save in [false, true] {
let legacy = LegacyPoolMeta {
version: POOL_META_V1_VERSION,
pools: Vec::new(),
dont_save,
};
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
let data = frame(payload.expect("three-field V1 payload must serialize"));
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
let mut trailing = data;
trailing.push(0xc0);
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
}
}
// This guard is only called for already-decoded replicas; malformed
// containers must fail selection rather than becoming repair sources.
let truncated = frame(vec![0x93, 0x01]);
assert!(matches!(
decode_pool_meta_replica(truncated),
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
));
}
#[test]
fn pool_meta_decode_supports_legacy_payload() {
let start_time = OffsetDateTime::now_utc();
@@ -30829,154 +30702,6 @@ mod pools_tests {
);
}
#[tokio::test]
async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
pool.disks.write().await[2..].fill(None);
let state = super::PoolMetaWriteState::default();
let data =
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
let mut transaction = state.arm_transaction();
save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut transaction,
)
.await
.expect_err("two of four disks cannot satisfy the identity write quorum");
drop(transaction);
for dir in &dirs {
assert!(
!dir.path()
.join(crate::disk::RUSTFS_META_BUCKET)
.join(POOL_META_IDENTITY_NAME)
.exists(),
"the rejected write must not create an identity replica"
);
}
state
.ensure_write_safe("retry after disks return")
.expect("a rejected inline write must not poison the startup transaction");
let ordinary_error = crate::config::com::save_config_with_opts(
pool.clone(),
"ordinary-config",
data.clone(),
&ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
},
)
.await
.expect_err("ordinary writes must still fail without quorum");
assert!(
matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
"ordinary callers must retain their existing quorum error"
);
*pool.disks.write().await = disks.into_iter().map(Some).collect();
let mut retry = state.arm_transaction();
save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut retry,
)
.await
.expect("the same transaction must succeed when the disks return");
retry.disarm();
drop(retry);
assert_eq!(
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
.await
.expect("read back the committed identity"),
data
);
state
.ensure_write_safe("after successful retry")
.expect("retry must leave writes enabled");
}
#[tokio::test]
async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
let state = super::PoolMetaWriteState::default();
let data =
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
let mut transaction = state.arm_transaction();
let written = save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"prepare_cas",
&mut transaction,
)
.await
.expect("the earlier phase must really commit");
pool.disks.write().await[2..].fill(None);
save_pool_meta_object_cas(
pool.clone(),
POOL_META_IDENTITY_NAME,
data.clone(),
&PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut transaction,
)
.await
.expect_err("the next phase must fail without a write quorum");
drop(transaction);
*pool.disks.write().await = disks.into_iter().map(Some).collect();
assert_eq!(
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
.await
.expect("the earlier commit must survive the rejected write"),
data
);
let error = state
.ensure_write_safe("after a committed phase")
.expect_err("a later rejection cannot discharge the earlier write");
assert_eq!(
error.pool_metadata_failure().expect("retain typed uncertainty").kind,
crate::error::PoolMetadataFailure::TransactionUnknown
);
assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
}
#[tokio::test]
async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
pool.disks.write().await[2..].fill(None);
let state = super::PoolMetaWriteState::default();
let mut transaction = state.arm_transaction();
save_pool_meta_object_cas(
pool,
POOL_META_IDENTITY_NAME,
vec![0x51; 2 * 1024 * 1024],
&PoolMetaCasToken::Missing,
&PoolMetaPersistenceFence::OfflineRecovery,
"identity_cas",
&mut transaction,
)
.await
.expect_err("the noninline write must fail without quorum");
drop(transaction);
assert!(
state.ensure_write_safe("after writer setup").is_err(),
"a failure after disk writers were created must remain fail-closed"
);
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum IdentityRepairFault {
Healthy,
-103
View File
@@ -853,8 +853,6 @@ pub(crate) struct DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
#[cfg(test)]
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
/// Caller API operations currently executing through the disk health wrapper.
api_waiting: AtomicU32,
/// Operations rejected because the disk was unavailable for the caller.
@@ -882,8 +880,6 @@ impl Default for DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex::new(None),
#[cfg(test)]
operation_timeout_override: std::sync::Mutex::new(None),
api_waiting: AtomicU32::new(0),
total_errors_availability: AtomicU64::new(0),
total_errors_timeout: AtomicU64::new(0),
@@ -1577,20 +1573,6 @@ impl Default for DiskHealthTracker {
}
}
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct ScopedDiskOperationTimeout {
metrics: Arc<DiskHealthMetricEpoch>,
previous: Option<Duration>,
}
#[cfg(test)]
impl Drop for ScopedDiskOperationTimeout {
fn drop(&mut self) {
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
}
}
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
/// This is similar to Go's xlStorageDiskIDCheck.
#[derive(Debug, Clone)]
@@ -1932,23 +1914,6 @@ impl LocalDiskWrapper {
self.health_check
}
#[cfg(test)]
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
// The metrics epoch follows this drive across clones and reconnects.
// Its scoped override cannot shorten unrelated healthy drives' I/O.
let previous = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.replace(timeout);
ScopedDiskOperationTimeout {
metrics: Arc::clone(&self.metrics),
previous,
}
}
/// Enable health monitoring after disk creation.
/// Used to defer health checks until after startup format loading completes.
pub fn enable_health_check(&self) {
@@ -2673,13 +2638,6 @@ impl LocalDiskWrapper {
}
return result;
}
#[cfg(test)]
let timeout_duration = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
// Execute the operation with timeout
let result = tokio::time::timeout(timeout_duration, operation).await;
@@ -6086,67 +6044,6 @@ mod tests {
assert!(health.offline_duration().is_none());
}
#[tokio::test]
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
let directories = [
tempfile::tempdir().expect("faulted drive"),
tempfile::tempdir().expect("healthy drive"),
];
let mut wrappers = Vec::new();
for directory in &directories {
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
wrappers.push(LocalDiskWrapper::new(disk, false));
}
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
Arc::clone(&wrappers[0].disk),
false,
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
);
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
tokio::time::pause();
let operation = || async {
tokio::time::sleep(Duration::from_secs(2)).await;
Ok(())
};
let (faulted, healthy) = tokio::join!(
reconnected.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
wrappers[1].track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
);
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
healthy.expect("healthy drive keeps its caller's deadline");
let shorter_started = tokio::time::Instant::now();
let shorter = wrappers[0]
.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_millis(500),
TimeoutHealthAction::IgnoreFailure,
)
.await;
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
assert!(shorter_started.elapsed() < Duration::from_secs(1));
wrappers[0]
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
.await
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
drop(scope);
reconnected
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
.await
.expect("dropping the scope restores reconnected callers' deadlines");
}
#[tokio::test]
async fn ignored_timeout_does_not_mark_drive_failure() {
let dir = tempfile::tempdir().expect("temp dir should be created");
+2 -464
View File
@@ -5,7 +5,6 @@ use super::{FdCache, FdCacheEntry, FdKey};
use std::fs::File;
use std::os::unix::fs::FileExt;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
FdKey {
@@ -53,11 +52,7 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
let (_directory, file) = fixture();
let cache = FdCache::new();
let entry = Arc::new(FdCacheEntry {
file,
len: 3,
invalidated: AtomicBool::new(false),
});
let entry = Arc::new(FdCacheEntry { file, len: 3 });
let old_generation = cache.generation();
cache.invalidate_exact("v", "part").await;
let current_generation = cache.generation();
@@ -130,7 +125,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
assert!(
cache.get(&key("v", "part", direct)).await.is_some(),
"new entries must survive the earlier volume invalidation"
"new entries must survive the earlier volume predicate"
);
}
cache.clear();
@@ -169,7 +164,6 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
let replacement = Arc::new(FdCacheEntry {
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
len: 11,
invalidated: AtomicBool::new(false),
});
for direct in [false, true] {
let key = key("v", "part", direct);
@@ -183,459 +177,3 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
assert_eq!(fresh.len, 11);
}
}
#[tokio::test]
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
let (_directory, file) = fixture();
let mut cache = FdCache::new();
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
cache.invalidate_under("v", "missing/prefix");
cache.invalidate_volume("v");
cache.invalidate_exact("v", "missing/prefix/part.1").await;
cache.clear();
assert!(
cache.maintenance_task.get().is_none(),
"empty-cache invalidations must not spawn maintenance"
);
// Populate a normal entry and verify matched vs unmatched filter gates directly,
// and ensure unmatched invalidations do not enqueue pending removals.
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
assert_eq!(cache.tracker.live_entries(), 1);
assert!(cache.tracker.may_match_volume("v"));
assert!(!cache.tracker.may_match_volume("other-volume"));
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
cache.invalidate_volume("other-volume");
cache.invalidate_under("v", "obj-2/0198a3f0");
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
assert!(
!cache
.tracker
.has_pending_invalidations
.load(std::sync::atomic::Ordering::SeqCst),
"unmatched invalidations must not enqueue maintenance removals"
);
assert!(cache.get(&cached_key).await.is_some());
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
// to per-volume unindexed_counts without degrading other volumes.
let other_vol_key = key("v2", "shallow/part.1", false);
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
let deep_key = key("v", deep_path, false);
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
assert_eq!(
cache
.tracker
.unindexed_counts
.iter()
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
.sum::<u32>(),
1
);
assert!(
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
"deep key in volume v must not disable prefix filtering in volume v2"
);
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
assert!(cache.get(&deep_key).await.is_none());
cache.invalidate_volume("v2");
assert!(cache.get(&other_vol_key).await.is_none());
// Replacing an invalidated key before maintenance drains pending_invalidations
// must release the old descriptor immediately and keep the replacement live
// even when a strong reference to the old CachedFdEntry shell survives until
// maintenance runs its Arc::ptr_eq check.
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
let old_weak = Arc::downgrade(&old_file);
cache.insert(cached_key.clone(), old_file).await;
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
cache.invalidate_under("v", "obj-1/0198a3f0");
assert!(cache.get(&cached_key).await.is_none());
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
assert!(
old_weak.upgrade().is_none(),
"replacing an entry must retire and drop the previous descriptor immediately"
);
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache
.tracker
.has_pending_invalidations
.load(std::sync::atomic::Ordering::SeqCst)
{
tokio::task::yield_now().await;
}
})
.await
.expect("maintenance should drain pending invalidations");
drop(held_old_shell);
assert!(
cache.get(&cached_key).await.is_some(),
"stale pending invalidation must not evict a newer replacement"
);
assert_eq!(
cache
.tracker
.unindexed_counts
.iter()
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
.sum::<u32>(),
0
);
// Dropping a cache while a live non-invalidated entry is present (even if a
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
// must retire the descriptor and release FdCacheTracker without cycles.
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
let live_drop_weak = Arc::downgrade(&live_drop_file);
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
let shutdown_weak = Arc::downgrade(&shutdown_file);
let tracker_weak = Arc::downgrade(&cache.tracker);
let cloned_moka = cache.cache.clone();
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
cache.invalidate_under("v", "shutdown");
if let Some(handle) = cache.maintenance_task.take() {
handle.abort();
let _ = handle.await;
}
drop(cache);
assert!(
live_drop_weak.upgrade().is_none(),
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
);
assert!(
shutdown_weak.upgrade().is_none(),
"dropping cache when maintenance is aborted must still release descriptors"
);
assert!(
tracker_weak.upgrade().is_none(),
"dropping cache must release FdCacheTracker without reference cycles"
);
drop(cloned_moka);
// Short-TTL expired entries (including mixed expired and fresh entries) must
// be retired by invalidate_under and clear() even though Moka's iterator skips
// expired entries.
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
let expired_weak = Arc::downgrade(&expired_file);
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
let fresh_weak = Arc::downgrade(&fresh_file);
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
ttl_cache.invalidate_under("v", "ttl-obj");
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
tokio::task::yield_now().await;
}
})
.await
.expect("invalidate_under must retire both expired and fresh matching descriptors");
assert_eq!(ttl_cache.tracker.live_entries(), 0);
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
ttl_cache
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
.await;
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
let clear_live_weak = Arc::downgrade(&clear_live_file);
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
ttl_cache.clear();
assert!(
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
"clear() must synchronously retire both TTL-expired and live descriptors"
);
assert_eq!(ttl_cache.tracker.live_entries(), 0);
}
#[tokio::test]
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
use std::collections::HashMap;
use std::sync::atomic::Ordering;
let (_directory, file) = fixture();
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
let paths = [
"a/b",
"a/b/part.1",
"a/b/dir/part.2",
"a/bc",
"a/bc/part.1",
"prefix/obj-0001.bin/0198a3f0/part.1",
"prefix/obj-0002.bin/0198a3f0/part.1",
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
];
let prefixes = [
"a",
"a/",
"a/b",
"a/b/",
"a/b/dir",
"a/bc",
"prefix",
"prefix/obj-0001.bin",
"prefix/obj-0001.bin/0198a3f0",
"d1/d2/d3/d4/d5/c_missing",
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
"nonexistent/prefix",
];
let mut candidate_keys = Vec::new();
for v in volumes {
for p in paths {
for direct in [false, true] {
candidate_keys.push(key(v, p, direct));
}
}
}
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
let mut next_u64 = || {
rng ^= rng << 13;
rng ^= rng >> 7;
rng ^= rng << 17;
rng
};
for step in 0..1000 {
let op = next_u64() % 100;
if op < 45 {
// Insert or replace (either fresh or stale generation)
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
let entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: step,
invalidated: AtomicBool::new(false),
});
let cur_gen = cache.generation();
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
if !stale {
oracle.insert(k, entry);
}
} else if op < 65 {
// Exact invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
let p = paths[(next_u64() as usize) % paths.len()];
cache.invalidate_exact(v, p).await;
oracle.remove(&key(v, p, false));
oracle.remove(&key(v, p, true));
} else if op < 88 {
// Prefix invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
cache.invalidate_under(v, raw_prefix);
let trimmed = raw_prefix.trim_end_matches('/');
oracle.retain(|k, _| {
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
});
} else if op < 97 {
// Volume invalidation
let v = volumes[(next_u64() as usize) % volumes.len()];
cache.invalidate_volume(v);
oracle.retain(|k, _| k.volume != v);
} else {
// Full clear
cache.clear();
oracle.clear();
}
// Verify lockstep equivalence against the reference oracle after every operation.
for k in &candidate_keys {
let got = cache.get(k).await;
let expected = oracle.get(k);
match (got, expected) {
(Some(g), Some(e)) => assert!(
Arc::ptr_eq(&g, e),
"step {step}: pointer mismatch for key ({}, {}, {})",
k.volume,
k.path,
k.direct
),
(None, None) => {}
(g, e) => panic!(
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
k.volume,
k.path,
k.direct,
g.is_some(),
e.is_some()
),
}
}
}
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
cache.clear();
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("clear must drain all live entries and pending invalidations");
assert_eq!(cache.tracker.live_entries(), 0);
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
use std::sync::atomic::{AtomicU64, Ordering};
let (_directory, file) = fixture();
let cache = Arc::new(FdCache::new());
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
// at `entry_shards[shard].lock()` after it has passed the initial generation
// check and incremented `volume_counts`, bump generation via an unmatched
// invalidation, and verify the post-`new()` recheck refuses publication and
// preserves the existing entry.
let seam_key = key("v-seam", "obj-1/part.1", false);
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
let stale_gen = cache.generation();
let stale_entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: 999_999,
invalidated: AtomicBool::new(false),
});
let blocked_insert = {
let hold = cache.tracker.entry_shards[entry_shard].lock();
let cache_clone = Arc::clone(&cache);
let seam_key_clone = seam_key.clone();
let task = tokio::spawn(async move {
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
});
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
assert!(
std::time::Instant::now() < deadline,
"insert never reached CachedFdEntry::new with the previous entry still live"
);
std::thread::yield_now();
}
cache.invalidate_under("v-seam", "unmatched-prefix");
drop(hold);
task
};
blocked_insert.await.expect("blocked insert task should complete");
let preserved = cache
.get(&seam_key)
.await
.expect("refused stale insert must not retire a valid previous entry");
assert!(Arc::ptr_eq(&preserved, &initial_entry));
cache.invalidate_volume("v-seam");
let floor_v1 = Arc::new(AtomicU64::new(0));
let floor_v2 = Arc::new(AtomicU64::new(0));
let mut handles = Vec::new();
for t in 0..8usize {
let cache = Arc::clone(&cache);
let file = Arc::clone(&file);
let floor_v1 = Arc::clone(&floor_v1);
let floor_v2 = Arc::clone(&floor_v2);
handles.push(tokio::spawn(async move {
for i in 0..250u64 {
let is_v1 = (t + i as usize).is_multiple_of(2);
let vol = if is_v1 { "v1" } else { "v2" };
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
let k = key(vol, &path, i.is_multiple_of(2));
let g = cache.generation();
let entry = Arc::new(FdCacheEntry {
file: Arc::clone(&file),
len: g,
invalidated: AtomicBool::new(false),
});
cache.insert_if_fresh(k.clone(), entry, g).await;
let min_gen = if is_v1 {
floor_v1.load(Ordering::SeqCst)
} else {
floor_v2.load(Ordering::SeqCst)
};
if let Some(hit) = cache.get(&k).await {
assert!(
hit.len >= min_gen,
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
hit.len
);
}
}
}));
}
for t in 0..4usize {
let cache = Arc::clone(&cache);
let floor_v1 = Arc::clone(&floor_v1);
let floor_v2 = Arc::clone(&floor_v2);
handles.push(tokio::spawn(async move {
for i in 0..250u64 {
let is_v1 = (t + i as usize).is_multiple_of(2);
let vol = if is_v1 { "v1" } else { "v2" };
match i % 4 {
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
1 => {
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
cache.invalidate_exact(vol, &p).await;
}
2 => {
let before = cache.generation();
cache.invalidate_volume(vol);
let floor = before + 1;
if is_v1 {
floor_v1.fetch_max(floor, Ordering::SeqCst);
} else {
floor_v2.fetch_max(floor, Ordering::SeqCst);
}
}
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
}
tokio::task::yield_now().await;
}
}));
}
for h in handles {
h.await.expect("worker task should succeed");
}
cache.clear();
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
})
.await
.expect("clear after concurrent stress must drain all live entries");
assert_eq!(cache.tracker.live_entries(), 0);
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
}
+31 -20
View File
@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
native_io(move || create_dir_all_std(&path)).await
}
fn is_dir_error(e: &io::Error) -> bool {
fn is_dir_error(path: &Path, e: &io::Error) -> bool {
e.raw_os_error() == Some(libc::EISDIR)
|| e.kind() == io::ErrorKind::IsADirectory
// macOS: remove_file on a directory returns EPERM
|| (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
|| is_windows_dir_access_denied(path, e)
}
/// Windows reports `remove_file` on a directory (or directory symlink) as
/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
/// failure without inspecting the entry itself.
#[cfg(windows)]
fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
use std::os::windows::fs::FileTypeExt;
e.kind() == io::ErrorKind::PermissionDenied
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| {
let file_type = metadata.file_type();
file_type.is_dir() || file_type.is_symlink_dir()
})
}
#[cfg(not(windows))]
fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
false
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
// Try remove_file first; fall back to remove_dir if it's a directory
let file_path = path.as_ref().to_path_buf();
match native_io(move || std::fs::remove_file(file_path)).await {
Ok(()) => Ok(()),
Err(e) if is_dir_error(&e) => {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::remove_dir(path)).await
}
Err(e) => Err(e),
}
let path = path.as_ref().to_path_buf();
native_io(move || remove_std(path)).await
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
// Try remove_file first; fall back to remove_dir_all if it's a directory
let file_path = path.as_ref().to_path_buf();
match native_io(move || std::fs::remove_file(file_path)).await {
let path = path.as_ref().to_path_buf();
native_io(move || match std::fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(e) if is_dir_error(&e) => {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::remove_dir_all(path)).await
}
Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
Err(e) => Err(e),
}
})
.await
}
#[tracing::instrument(level = "debug", skip_all)]
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
// Try remove_file first; fall back to remove_dir if it's a directory
match std::fs::remove_file(path.as_ref()) {
let path = path.as_ref();
match std::fs::remove_file(path) {
Ok(()) => Ok(()),
Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
Err(e) => Err(e),
}
}
File diff suppressed because it is too large Load Diff
+10 -4
View File
@@ -198,7 +198,12 @@ pub(super) fn create_local_inline_rollback_backup(
return Err(err);
}
} else if sync && !old_metadata.is_empty() {
std::fs::File::open(&backup_path)?.sync_data()?;
// Windows only flushes a handle opened for writing.
#[cfg(windows)]
let backup = std::fs::OpenOptions::new().write(true).open(&backup_path)?;
#[cfg(not(windows))]
let backup = std::fs::File::open(&backup_path)?;
backup.sync_data()?;
}
Ok(backup_path)
}
@@ -1594,12 +1599,13 @@ impl LocalDisk {
);
}
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
#[cfg(target_os = "linux")]
#[cfg(any(target_os = "linux", windows))]
let payload_guard = if multipart_publication.is_some() {
Some(
os::multipart_payload_commit_guard(
&commit.directory_guard,
src_data_path.parent().ok_or(DiskError::InvalidPath)?,
commit.publication_root,
commit.mutation_lease.clone(),
)
.await
@@ -1608,9 +1614,9 @@ impl LocalDisk {
} else {
None
};
#[cfg(target_os = "linux")]
#[cfg(any(target_os = "linux", windows))]
let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
#[cfg(not(target_os = "linux"))]
#[cfg(not(any(target_os = "linux", windows)))]
let directory_guard = &commit.directory_guard;
if let Err(err) = os::rename_all_with_commit_guard(
src_data_path,
@@ -1002,17 +1002,21 @@ pub(super) fn inspection_with_markers(
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
let mut map = PREPARED_DELETE_RECEIPTS.lock();
if map.len() >= 4096
&& let Some(evict_key) = map.keys().next().cloned()
{
map.remove(&evict_key);
}
map.insert(rollback, entry);
}
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
PREPARED_DELETE_RECEIPTS.remove(rollback)
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
}
impl LocalDisk {
@@ -270,17 +270,21 @@ struct ActiveMultipartSourceEntry {
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
}
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
let source = source.as_ref();
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
return;
};
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
if map.len() >= 4096
&& let Some(evict_key) = map.keys().next().cloned()
{
map.remove(&evict_key);
}
map.insert(
source,
ActiveMultipartSourceEntry {
raw,
@@ -300,50 +304,56 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
}
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
let (raw, fi, head_stamp) =
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
let (raw, fi, head_stamp) = {
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let entry = map.get(source)?;
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return None;
}
Some((raw, fi))
}
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
return false;
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return false;
}
true
}
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
let Some((head_stamp, clean)) =
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
.lock()
.get(source)
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
else {
return false;
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return false;
}
clean
}
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
entry.unsettled_parts.insert(part_number);
entry.parts.remove(&part_number);
});
}
}
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
.with_get(source, |entry| entry.fi.data_dir)
.flatten()
.lock()
.get(source)
.and_then(|entry| entry.fi.data_dir)
.filter(|dir| !dir.is_nil())
else {
return;
@@ -352,9 +362,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
return;
};
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
entry.parts.insert(part.number, (part_stamp, part));
});
}
}
pub(super) fn finish_active_multipart_part_transaction(
@@ -363,29 +373,34 @@ pub(super) fn finish_active_multipart_part_transaction(
committed: bool,
retired_existing_txn: bool,
) {
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let Some(part_number) = part_number.filter(|_| committed) else {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
map.remove(source);
return;
};
if !retired_existing_txn {
return;
}
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
if let Some(entry) = map.get_mut(source)
&& !entry.unsettled_parts.remove(&part_number)
{
map.remove(source);
}
}
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
.with_get(source, |entry| {
if !entry.unsettled_parts.is_empty() {
return None;
}
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
let (expected_stamp, part) = entry.parts.get(&part_number)?;
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
})
.flatten()?;
let (head_stamp, data_dir, expected_stamp, part) = {
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
let entry = map.get(source)?;
if !entry.unsettled_parts.is_empty() {
return None;
}
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
let (expected_stamp, part) = entry.parts.get(&part_number)?;
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
};
if !stamp_matches_active_disk(source, head_stamp) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
return None;
}
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
@@ -396,11 +411,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
}
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
}
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
}
/// Canonical source mutations are admitted under SourceRW at the syscall,
@@ -1512,7 +1527,7 @@ impl LocalDisk {
}
Err(error) => return Err(error),
};
let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
if !parts.is_empty() {
let anchor_path = directory.join(RETIREMENT_ANCHOR);
let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
@@ -1545,7 +1560,13 @@ impl LocalDisk {
}
// An anchor-less empty DD is only rmdir'ed. A new child makes
// this fail closed; no recursive removal can consume its bytes.
// A Windows pin refuses DELETE sharing, so the leaf pin is
// released first; the object pin still holds the parent.
#[cfg(windows)]
drop(directory_pin);
std::fs::remove_dir(&directory)?;
#[cfg(not(windows))]
drop(directory_pin);
if sync {
os::fsync_dir_std(&object_path)?;
}
@@ -1921,7 +1942,6 @@ impl NativeMultipartPublication {
*self.validated_prepared.lock() = Some(validated);
Ok(())
}
#[cfg(target_os = "linux")]
pub(super) fn prepare_and_install_head_on_thread(
&self,
new_metadata: &[u8],
File diff suppressed because it is too large Load Diff
@@ -1899,6 +1899,10 @@ mod tests {
&ObjectOptions {
version_id: Some(version_id.to_string()),
mod_time: earlier(version.mod_time),
// Without the object lock the rename waits for every
// drive, so no early-ACK tail is still writing the copies
// removed below.
no_lock: true,
..versioned
},
))
+3 -106
View File
@@ -318,21 +318,6 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
}
}
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
match rebalance_error_source(err) {
Error::Lock(err) => is_rebalance_transient_lock_error(err),
// No metadata was written when lock admission failed. Each attempt
// must acquire its quorum again before loading or saving this run.
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket,
object,
..
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
_ => false,
}
}
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
if err.kind() == std::io::ErrorKind::TimedOut {
return true;
@@ -400,7 +385,9 @@ where
match result {
Ok(value) => return Ok(value),
Err(err) => {
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
if attempt.saturating_add(1) >= max_attempts.max(1)
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
{
return Err(err);
}
#[cfg(test)]
@@ -732,82 +719,6 @@ mod error_source_tests {
}
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
let mut attempts = 0;
retry_rebalance_metadata_access(None, 3, || {
attempts += 1;
std::future::ready(if attempts == 1 {
resolve_rebalance_meta_save_result(
Err(rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)),
"stop_rebalance",
)
} else {
Ok(())
})
})
.await
.expect("a later quorum-protected admission must be allowed to succeed");
assert_eq!(attempts, 2);
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
for max_attempts in [0, 1, 3] {
let mut attempts = 0;
let result = retry_rebalance_metadata_access(None, max_attempts, || {
attempts += 1;
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)))
})
.await;
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
assert_eq!(attempts, max_attempts.max(1));
assert!(matches!(
rebalance_error_source(&error),
Error::NamespaceLockQuorumUnavailable {
mode: "write",
required: 1,
achieved: 0,
..
}
));
}
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
let mut attempts = 0;
let result = retry_rebalance_metadata_access(None, 3, || {
attempts += 1;
std::future::ready(Err::<(), _>(if attempts == 1 {
rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)
} else {
Error::OperationCanceled
}))
})
.await;
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
assert!(matches!(result, Err(Error::OperationCanceled)));
}
#[tokio::test]
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
for err in [
@@ -820,20 +731,6 @@ mod error_source_tests {
required: 3,
achieved: 2,
},
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: "data-bucket".to_string(),
object: REBAL_META_NAME.to_string(),
required: 1,
achieved: 0,
},
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
object: "other-metadata.bin".to_string(),
required: 1,
achieved: 0,
},
Error::other("stale rebalance run rejected: lock acquisition timed out"),
Error::other("rebalance distributed run fence lost"),
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
+6 -36
View File
@@ -4696,17 +4696,6 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
}
// Reusing a held writer additionally requires the same quorum, so repeated
// clients must not give different weight to an otherwise identical domain.
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
!left.is_empty()
&& left.len() == right.len()
&& left.iter().all(|client| {
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
})
}
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5019,11 +5008,6 @@ impl DiskHealthEntry {
#[error("committed system metadata write requires repair by its owning writer")]
pub(crate) struct SystemMetadataRepairRequired;
/// An inline metadata write was rejected before staging or commit began.
#[derive(Debug, thiserror::Error)]
#[error("system metadata write rejected before staging: {0}")]
pub(crate) struct SystemMetadataWriteNotStarted(#[source] pub(crate) Error);
impl SetDisks {
pub(in crate::set_disk) async fn persist_delete_marker_purge(
&self,
@@ -5661,9 +5645,9 @@ impl SetDisks {
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
// Distributed lock clients receive the raw ObjectKey; the pool
// namespace is not part of the key stored by their backends.
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
(true, true) => {
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
}
_ => false,
}
}
@@ -9898,7 +9882,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
async fn namespace_lock_domain_includes_pool_namespace() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9937,25 +9921,11 @@ mod tests {
"sets in the same pool share the object namespace lock domain"
);
assert!(
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
"different pools using identical backends lock the same raw object key"
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
"different pool namespaces must not be deduplicated solely by identical clients"
);
}
#[test]
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
assert!(!same_distributed_lock_quorum(
&[a.clone(), a.clone(), b.clone()],
&[a.clone(), b.clone(), b]
));
assert!(!same_distributed_lock_quorum(&[], &[]));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn streaming_reader_holds_read_lock_until_eof() {
+13 -23
View File
@@ -375,10 +375,7 @@ impl SetDisks {
return None;
}
// A failed remote reconnect must retain its existing recovery monitor.
if disk.is_local() {
let _ = disk.close().await;
}
let _ = disk.close().await;
}
let endpoint = self.set_endpoints.get(i)?;
@@ -500,26 +497,19 @@ impl SetDisks {
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
if !new_disk.is_local() {
let displaced = {
let mut disks = self.disks.write().await;
let slot = &mut disks[disk_idx];
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
(None, None) => true,
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
_ => false,
};
// A GET probe or another reconnect may have filled this slot while
// format I/O was pending. Only the successful publisher owns the
// displaced monitor; a failed or superseded probe must retain it.
if unchanged {
new_disk.enable_health_check();
slot.replace(new_disk)
} else {
None
}
let mut disks = self.disks.write().await;
let slot = &mut disks[disk_idx];
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
(None, None) => true,
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
_ => false,
};
if let Some(disk) = displaced {
let _ = disk.close().await;
// A GET probe or another reconnect may have filled this slot while
// format I/O was pending. Do not replace its handle or start a
// second health monitor for an unpublished handle.
if unchanged {
new_disk.enable_health_check();
*slot = Some(new_disk);
}
return false;
}
+3 -46
View File
@@ -57,6 +57,7 @@ use super::object::{
object_transaction_fencing_requested, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence,
};
use crate::api::config::storageclass;
#[cfg(test)]
use crate::bucket::metadata_sys::ObjectLockConfigState;
use crate::bucket::quota::reservation;
use crate::bucket::replication::ReplicationStatusType;
@@ -4113,40 +4114,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
)
.await?;
if !opts.versioned
&& !crate::bucket::utils::is_meta_bucketname(bucket)
&& let Some(snapshot) = opts.object_lock_config_snapshot.as_deref()
&& !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)
{
// Completion replaces the null version, not necessarily the latest
// version inspected by the request-level Object Lock check.
let current = self
.get_object_info(
bucket,
object,
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
no_lock: true,
metadata_cache_safe: false,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
..Default::default()
},
)
.await;
match current {
Ok(existing) => {
if check_object_lock_for_deletion_with_state(snapshot.state(), &existing, false)?.is_some() {
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
}
}
// A null delete marker has no protected payload to replace.
Err(StorageError::MethodNotAllowed) => {}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
}
}
if opts.data_movement
&& opts.http_preconditions.is_some()
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
@@ -10093,7 +10060,7 @@ mod tests {
// Real disk callbacks and health probes must share wall-clock deadlines.
// Advancing Tokio time can expire unrelated healthy-drive probes.
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "mpu-quorum-success-physical";
@@ -10120,8 +10087,6 @@ mod tests {
local.observe_multipart_cleanup_for_test(&upload_path)
}).collect::<Vec<_>>();
let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
// Fault only the held leg; healthy disks keep their normal I/O budgets.
let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
@@ -10278,15 +10243,7 @@ mod tests {
let size = copy_info.get_actual_size().expect("actual source size");
copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
.await.expect("cross-key Copy cannot wait on the old successful logical owner")
.unwrap_or_else(|error| {
let health = disks.iter().enumerate().map(|(slot, disk)| {
let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
let metrics = local.metrics_snapshot();
(slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
}).collect::<Vec<_>>();
panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
});
.await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
drop(copy_info);
let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
let mut actual = Vec::new();
+5 -97
View File
@@ -4526,16 +4526,7 @@ impl SetDisks {
"Set disk write quorum unavailable"
);
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
let error = to_object_err(write_err.into(), vec![bucket, object]);
if direct_inline_commit
&& is_meta_bucketname(bucket)
&& opts.write_completion == crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair
{
// Inline writer setup only checks disk availability;
// no shard or metadata write has been dispatched yet.
return Err(Error::other(crate::set_disk::SystemMetadataWriteNotStarted(error)));
}
return Err(error);
return Err(to_object_err(write_err.into(), vec![bucket, object]));
}
return Err(Error::other(format!("not enough disks to write: {errors:?}")));
@@ -4967,20 +4958,7 @@ impl SetDisks {
);
}
// A write without a version id to a key that is not versioning-Enabled
// replaces the key's null version in place, so the WORM gate must judge
// that version too. The request-layer pre-check reads the latest
// version, which is not the one replaced when a UUID version or a
// delete marker sits above a retained null version.
let worm_gate_version_id = opts.version_id.clone().or_else(|| {
(!opts.versioned
&& opts
.object_lock_config_snapshot
.as_ref()
.is_some_and(|snapshot| !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)))
.then(|| Uuid::nil().to_string())
});
if let Some(version_id) = worm_gate_version_id.as_deref()
if let Some(version_id) = opts.version_id.as_deref()
&& !is_meta_bucketname(bucket)
{
let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer();
@@ -5000,10 +4978,9 @@ impl SetDisks {
.await;
match current {
Ok(existing) => {
let object_lock_config = opts
.object_lock_config_snapshot
.as_deref()
.ok_or_else(|| Error::other("destructive PUT is missing its Object Lock configuration snapshot"))?;
let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
})?;
// The WORM gate protects the locked version from local
// overwrites; an authorized replication write passes it
// only when the LWW merge below will judge every
@@ -5034,8 +5011,6 @@ impl SetDisks {
}
}
}
// A null delete marker has no protected payload to replace.
Err(StorageError::MethodNotAllowed) if version_id == Uuid::nil().to_string() => {}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
}
@@ -9329,60 +9304,6 @@ impl SetDisks {
}
impl SetDisks {
/// Judge the version a versionless destructive delete removes.
///
/// For a key that is not versioning-`Enabled`, a delete without a version
/// id removes the `null` version, while the preceding Object Lock check
/// judged the key's latest entry. When a UUID version or delete marker
/// sits above a retained `null` version, the two differ and the retained
/// version must be judged on its own. The caller holds the object's write
/// lock. Plain unversioned keys and versioning-`Enabled` keys keep a single
/// lookup: their latest entry is the version removed.
///
/// The lookup future is boxed from a synchronous wrapper so it leaves the
/// delete futures' frames: `delete_object` already nests the boxed batch
/// path, and inlining a second `get_object_info` overflowed the 4 MiB test
/// stack on Linux.
fn check_null_version_object_lock_delete<'a>(
&'a self,
bucket: &'a str,
object: &'a str,
latest: &'a ObjectInfo,
opts: &'a ObjectOptions,
) -> impl Future<Output = Result<()>> + Send + 'a {
Box::pin(self.check_null_version_object_lock_delete_inner(bucket, object, latest, opts))
}
async fn check_null_version_object_lock_delete_inner(
&self,
bucket: &str,
object: &str,
latest: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<()> {
if is_meta_bucketname(bucket)
|| opts.version_id.is_some()
|| set_disk_delete_creates_delete_marker(opts)
|| opts.transition.expire_restored
|| latest.version_id.is_none_or(|version| version.is_nil())
{
return Ok(());
}
let null_opts = ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
no_lock: true,
metadata_cache_safe: false,
..opts.clone()
};
match self.get_object_info(bucket, object, &null_opts).await {
Ok(existing) => check_object_lock_delete(&self.ctx, bucket, object, &existing, &null_opts).await,
// A null delete marker has no protected payload to remove.
Err(StorageError::MethodNotAllowed) => Ok(()),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
Err(err) => Err(err),
}
}
/// Read current metadata without cache admission or read-side repair. The
/// outer batch owner already holds the sorted namespace write-lock set.
pub(crate) async fn conditional_delete_current_snapshot(
@@ -10101,17 +10022,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
del_errs[i] = Some(err);
continue;
}
// Not gated on `source_missing`: with a delete marker on top the
// latest-version lookup reports the key as missing while the null
// version below it may still be retained.
if object_lock_check_required
&& let Err(err) = self
.check_null_version_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts)
.await
{
del_errs[i] = Some(err);
continue;
}
if opts.tier_delete_journal_api.is_some()
&& !source_missing
@@ -11184,8 +11094,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
opts.precondition_check(&goi)?;
check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?;
}
self.check_null_version_object_lock_delete(bucket, object, &goi, &opts)
.await?;
let delete_marker_purge_candidate = opts
.version_id
.as_deref()
-171
View File
@@ -3692,177 +3692,6 @@ mod tests {
);
}
// The RPC lock service uses the raw ObjectKey and one backend per node,
// even when the requesting sets carry different pool namespace strings.
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
use rustfs_lock::client::{LockClient, local::LocalClient};
let clients = || {
(0..2)
.map(|_| {
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
})
.collect::<Vec<_>>()
};
let first = clients();
let second = if shared { first.clone() } else { clients() };
let mut pools = source.pools.clone();
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
for set in &mut Arc::make_mut(pool).disk_set {
Arc::make_mut(set).lockers = lockers.clone();
}
}
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
Arc::new(ECStore {
id: Uuid::new_v4(),
disk_map: Default::default(),
pools,
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
rebalance_meta: tokio::sync::RwLock::new(None),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: tokio::sync::Mutex::new(state),
ctx: Arc::clone(&source.ctx),
bucket_fence_registry: Arc::default(),
})
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
let first_lock = first
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("first pool lock");
let second_lock = second
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("second pool lock");
let guard = first_lock
.get_write_lock(std::time::Duration::from_secs(2))
.await
.expect("first writer");
assert!(
second_lock
.get_write_lock(std::time::Duration::from_millis(150))
.await
.is_err(),
"different pool namespaces still conflict on the same backend and object key"
);
drop(guard);
second_lock
.get_write_lock(std::time::Duration::from_secs(3))
.await
.expect("second writer should acquire after the first releases");
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let (item, error) = tokio::time::timeout(
std::time::Duration::from_secs(3),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
dry_run: true,
..Default::default()
},
),
)
.await
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
.expect("pool metadata heal inspection should return a result");
assert!(error.is_none(), "inspection should succeed: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
let missing = remove_pool_meta_shard(&store, 1).await;
let (item, error) = tokio::time::timeout(
std::time::Duration::from_secs(10),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
..Default::default()
},
),
)
.await
.expect("cross-pool repair must not reenter the shared writer")
.expect("pool metadata repair should return a result");
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
assert!(
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
"the selected missing shard must be restored"
);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
let lock = target
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("target lock");
let guard = lock
.get_write_lock(std::time::Duration::from_secs(2))
.await
.expect("hold target lock");
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
let heal_store = Arc::clone(&store);
let mut heal = tokio::spawn(async move {
heal_store
.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
pool: Some(1),
dry_run: true,
..Default::default()
},
)
.await
});
tokio::time::timeout(
std::time::Duration::from_secs(3),
barrier.wait_until_external_heal_target_lock_attempted(),
)
.await
.expect("inspection must attempt the independent target lock");
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
drop(guard);
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
.await
.expect("inspection should progress after target lock release")
.expect("heal task should not panic")
.expect("pool metadata heal inspection should return a result");
assert!(error.is_none(), "inspection should succeed: {error:?}");
assert_eq!(item.object, POOL_META_NAME);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
-279
View File
@@ -540,7 +540,6 @@ impl ECStore {
// failures can still spawn recovery probes for peers that come up
// after this node.
let (mut disks, errs) = init_format::init_disks(
&instance_ctx,
&pool_eps.endpoints,
&DiskOption {
cleanup: true,
@@ -27353,282 +27352,4 @@ mod tests {
.expect("recovery must preserve the local source object");
assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
}
#[cfg(feature = "test-util")]
async fn seed_retained_null_version(
store: &Arc<crate::store::ECStore>,
bucket: &str,
object: &str,
protection: &str,
latest_is_marker: bool,
) {
let metadata = if protection == "legal-hold" {
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
} else {
HashMap::from([
("x-amz-object-lock-mode".to_string(), protection.to_string()),
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
])
};
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
store
.put_object(
bucket,
object,
&mut original,
&ObjectOptions {
user_defined: metadata,
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
..Default::default()
},
)
.await
.expect("seed retained null version");
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
store
.put_object(
bucket,
object,
&mut latest,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("seed newer unprotected UUID version");
if latest_is_marker {
store
.delete_object(
bucket,
object,
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("create a UUID delete marker above the retained null version");
}
}
#[cfg(feature = "test-util")]
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
let mut reader = store
.get_object_reader(
bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
},
)
.await
.expect("retained null version must remain readable");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("read retained null version");
assert_eq!(body, b"protected null version");
}
/// A versionless delete removes the null version, so the Object Lock check
/// must judge that version even when a UUID version or delete marker is
/// the key's latest entry (rustfs/backlog#3017).
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn null_version_object_lock_delete_checks_actual_target() {
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
let temp = tempfile::tempdir().expect("create null-version delete store");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("create Object Lock bucket");
let configs = [
VersioningConfiguration::default(),
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
];
for (case, config) in configs.into_iter().enumerate() {
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
for latest_is_marker in [false, true] {
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
let (_, errors) = store
.delete_objects(
&bucket,
vec![ObjectToDelete {
object_name: object.clone(),
..Default::default()
}],
ObjectOptions {
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
..Default::default()
},
)
.await;
assert!(
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
"batch delete must judge the retained null version: {object}: {errors:?}"
);
let error = store
.delete_object(
&bucket,
&object,
ObjectOptions {
version_suspended: true,
..Default::default()
},
)
.await
.expect_err("suspended single delete must also judge the retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
assert_retained_null_version_body(&store, &bucket, &object).await;
if protection == "GOVERNANCE" {
let (_, errors) = store
.delete_objects(
&bucket,
vec![ObjectToDelete {
object_name: object.clone(),
..Default::default()
}],
ObjectOptions {
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
bypass_governance: true,
}),
..Default::default()
},
)
.await;
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
assert!(
store
.get_object_reader(
&bucket,
&object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(Uuid::nil().to_string()),
..Default::default()
}
)
.await
.is_err(),
"authorized deletion must make the retained null payload unreadable"
);
}
}
}
}
}
/// A versionless PUT or multipart completion replaces the null version in
/// place, so the commit-time WORM gate must judge that version even when a
/// UUID version or delete marker is the key's latest entry
/// (rustfs/backlog#3019).
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
let temp = tempfile::tempdir().expect("create null-version overwrite store");
let (_ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
store
.make_bucket(
&bucket,
&MakeBucketOptions {
lock_enabled: true,
..Default::default()
},
)
.await
.expect("create Object Lock bucket");
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
for latest_is_marker in [false, true] {
let object = format!("{protection}-{latest_is_marker}");
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
let error = store
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
.await
.expect_err("PUT must reject replacement of a retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
let upload = store
.clone()
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
.await
.expect("initiate multipart upload");
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
let part = store
.clone()
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
.await
.expect("stage multipart replacement");
let error = store
.clone()
.complete_multipart_upload(
&bucket,
&object,
&upload.upload_id,
vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&ObjectOptions::default(),
)
.await
.expect_err("multipart completion must reject replacement of a retained null version");
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
assert_retained_null_version_body(&store, &bucket, &object).await;
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
store
.put_object(
&bucket,
&object,
&mut next_version,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("a new UUID version must remain allowed");
assert_retained_null_version_body(&store, &bucket, &object).await;
}
}
}
}
@@ -318,7 +318,10 @@ mod multipart_copy_heal_tests {
let mut corrupt = original.clone();
corrupt[0] ^= 0x80;
std::fs::write(&shard, &corrupt).expect("actual bitrot corruption in shared physical shard");
std::fs::File::open(&shard)
// Windows only flushes a handle opened for writing.
std::fs::OpenOptions::new()
.write(true)
.open(&shard)
.expect("open corrupted actual shard")
.sync_all()
.expect("persist actual fault bytes");
+2 -109
View File
@@ -44,32 +44,12 @@ const LEGACY_FORMAT_READ_CONCURRENCY: usize = 16;
const LEGACY_FORMAT_BASE_MAX_BYTES: usize = 16 * 1024;
const LEGACY_FORMAT_MAX_BYTES_PER_DISK: usize = 64;
pub async fn init_disks(
instance_ctx: &InstanceContext,
eps: &Endpoints,
opt: &DiskOption,
) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
let mut futures = Vec::with_capacity(eps.as_ref().len());
let local_disk_map = instance_ctx.local_disk_map();
let local_disks = local_disk_map.read().await;
for ep in eps.as_ref().iter() {
// Foundation disks may already serve RPC writes. Opening them again
// with startup cleanup can move live staging out from under a writer.
let registered = if ep.is_local {
local_disks.get(&ep.to_string()).cloned().flatten()
} else {
None
};
futures.push(async move {
match registered {
Some(disk) if disk.endpoint() == *ep => Ok(disk),
Some(_) => Err(DiskError::InconsistentDisk),
None => new_disk(ep, opt).await,
}
});
futures.push(new_disk(ep, opt));
}
drop(local_disks);
let mut res = Vec::with_capacity(eps.as_ref().len());
let mut errors = Vec::with_capacity(eps.as_ref().len());
@@ -1050,7 +1030,6 @@ mod tests {
}
let (disks, errors) = init_disks(
&InstanceContext::new(),
&Endpoints::from(endpoints),
&DiskOption {
cleanup: false,
@@ -1070,91 +1049,6 @@ mod tests {
(temp_dir, disks)
}
#[tokio::test]
async fn init_disks_preserves_live_staging_on_registered_local_disks() {
let (temp_dir, disks) = local_disks(1).await;
let registered = disks[0].as_ref().expect("registered disk should exist");
let instance_ctx = Arc::new(InstanceContext::new());
record_local_disks(&instance_ctx, vec![registered.clone()]).await;
let staging = temp_dir.path().join("disk-0/.rustfs.sys/tmp/in-flight/xl.meta");
tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
.await
.expect("create staging directory");
tokio::fs::write(&staging, b"in-flight pool metadata")
.await
.expect("stage metadata through the registered disk");
let (initialized, errors) = init_disks(
&instance_ctx,
&Endpoints::from(vec![registered.endpoint()]),
&DiskOption {
cleanup: true,
health_check: false,
},
)
.await;
assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
assert_eq!(
tokio::fs::read(&staging)
.await
.expect("startup must preserve live RPC staging"),
b"in-flight pool metadata"
);
assert!(Arc::ptr_eq(registered, initialized[0].as_ref().expect("initialized disk should exist")));
}
#[tokio::test]
async fn init_disks_cleans_stale_staging_without_a_registered_disk() {
let temp_dir = tempfile::tempdir().expect("create disk root");
let endpoint =
Endpoint::try_from(temp_dir.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
let staging = temp_dir.path().join(".rustfs.sys/tmp/stale/xl.meta");
tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
.await
.expect("create stale staging directory");
tokio::fs::write(&staging, b"stale pool metadata")
.await
.expect("write stale staging");
let (disks, errors) = init_disks(
&InstanceContext::new(),
&Endpoints::from(vec![endpoint]),
&DiskOption {
cleanup: true,
health_check: false,
},
)
.await;
assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
assert!(disks[0].is_some());
assert!(!staging.exists(), "a fresh disk open must still clear stale staging");
}
#[tokio::test]
async fn init_disks_rejects_registered_disk_topology_mismatch() {
let (_temp_dir, disks) = local_disks(1).await;
let registered = disks[0].as_ref().expect("registered disk should exist");
let instance_ctx = Arc::new(InstanceContext::new());
record_local_disks(&instance_ctx, vec![registered.clone()]).await;
let mut endpoint = registered.endpoint();
endpoint.set_pool_index(1);
let (initialized, errors) = init_disks(
&instance_ctx,
&Endpoints::from(vec![endpoint]),
&DiskOption {
cleanup: true,
health_check: false,
},
)
.await;
assert!(initialized[0].is_none(), "a registered disk must not be reopened in a different slot");
assert_eq!(errors[0], Some(DiskError::InconsistentDisk));
}
async fn format_bytes(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
let mut snapshots = Vec::with_capacity(disks.len());
for disk in disks {
@@ -1864,7 +1758,6 @@ mod tests {
disk.set_disk_id(None).await.expect("test disk ID should be reset");
}
let (prewarm_disks, prewarm_errors) = init_disks(
&InstanceContext::new(),
&prewarm_endpoints,
&DiskOption {
cleanup: false,
-1
View File
@@ -3295,7 +3295,6 @@ mod tests {
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
let (mut disks, errs) = init_disks(
&crate::runtime::global::current_ctx(),
&endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
&DiskOption {
cleanup: true,
-27
View File
@@ -1210,34 +1210,7 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
check_complete_multipart_args(bucket, object, upload_id)?;
let encoded_object = encode_dir_object_ref(object);
let object = encoded_object.as_ref();
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
// The set-level commit judges the null version this completion
// replaces; give it the same authoritative snapshot a PUT carries.
let expected_incarnation_id = opts
.expected_bucket_incarnation_id
.ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
let lifecycle_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
Some(snapshot) => Arc::clone(snapshot),
None => {
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
.await?
}
};
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
return Err(Error::other(
"Object Lock snapshot does not hold valid target bucket generation and configuration fences",
));
}
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
if !self.single_pool() {
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
}
+2 -81
View File
@@ -925,13 +925,6 @@ impl MetaCacheEntriesSorted {
const METACACHE_STREAM_VERSION: u8 = 2;
const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
/// Upper bound on one incremental reservation while accumulating a single
/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
/// up to this bound so a legitimately large xl.meta does not pay one small
/// read per chunk, while a corrupt length prefix still cannot force a
/// reservation larger than the payload that has already arrived (see
/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
/// msgpack `false`: the entry-stream terminator, and the only byte a legacy
/// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -1164,14 +1157,13 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
// A whole xl.meta can legitimately exceed the limit on its individual
// msgpack values. Grow only as payload arrives, so a corrupt length
// prefix cannot cause a speculative allocation of the advertised size.
let mut chunk = METACACHE_READ_CHUNK_SIZE;
while self.buf.len() < read_size {
let start = self.buf.len();
let extra = if lookahead {
let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
(read_size - start).clamp(min_chunk, chunk)
(read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
} else {
(read_size - start).min(chunk)
(read_size - start).min(METACACHE_READ_CHUNK_SIZE)
};
if let Err(e) = self.buf.try_reserve(extra) {
let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1202,7 +1194,6 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
return Err(err);
}
}
chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
}
self.pos = read_size;
@@ -2164,76 +2155,6 @@ mod tests {
}
}
/// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
/// decode completely even when the aggregate stream exceeds the per-value
/// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
/// element.
#[tokio::test]
async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
// ~2.9 KiB of metadata per entry matches the per-entry cost observed in
// the #9120 production report; 6,000 entries push the stream past 16 MiB.
const ENTRIES: usize = 6_000;
let metadata = vec![0xab; 2_880];
let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
.map(|i| MetaCacheEntry {
name: format!("bigdir/object-{i:05}"),
metadata: metadata.clone(),
..Default::default()
})
.collect();
let mut output = Cursor::new(Vec::new());
let mut writer = MetacacheWriter::new(&mut output);
writer.write(&entries).await.expect("wide listing must encode");
writer.close().await.expect("stream must close");
let bytes = output.into_inner();
assert!(
bytes.len() > 16 << 20,
"fixture must exceed the per-value limit, got {} bytes",
bytes.len()
);
let mut reader = MetacacheReader::new(Cursor::new(bytes));
let decoded = reader.read_all().await.expect("wide listing must decode");
assert_eq!(decoded, entries);
}
/// The #9120 failure shape seen in production: one entry whose metadata is
/// an xl.meta that grew past 16 MiB with its version history, listed among
/// ordinary neighbors and skipped past without losing the framing.
#[tokio::test]
async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
let small = |i: u8| MetaCacheEntry {
name: format!("bigdir/object-{i}"),
metadata: vec![i; 64],
..Default::default()
};
let oversized = MetaCacheEntry {
name: "bigdir/huge-versions".to_string(),
metadata: vec![0xcd; (16 << 20) + 12_345],
..Default::default()
};
let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
let mut output = Cursor::new(Vec::new());
let mut writer = MetacacheWriter::new(&mut output);
writer.write(&entries).await.expect("mixed listing must encode");
writer.close().await.expect("stream must close");
let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
assert_eq!(
reader.peek().await.expect("first entry").expect("present"),
entries[0],
"an ordinary entry before the oversized one must decode"
);
reader.skip(1).await.expect("skip the first entry");
assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
assert_eq!(
reader.peek().await.expect("oversized entry").expect("present"),
oversized,
"an entry past the per-value limit must round-trip whole"
);
reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
}
#[test]
fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
let version_id = Uuid::new_v4();
-30
View File
@@ -128,20 +128,6 @@ impl Error {
Error::TransientSkip { message: message.into() }
}
/// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
/// Callers must cut their own concurrency instead of retrying at full speed.
pub(crate) fn is_slow_down(&self) -> bool {
match self {
Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
message.contains("Please reduce your request rate")
}
Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
_ => false,
}
}
/// Whether a heal operation can be retried without changing its inputs.
pub(crate) fn is_recoverable_heal(&self) -> bool {
match classify_heal_error(self) {
@@ -463,22 +449,6 @@ mod tests {
use super::Error;
use crate::heal::{DiskError, EcstoreError};
#[test]
fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
assert!(
Error::TaskExecutionFailed {
message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
}
.is_slow_down()
);
assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
}
#[test]
fn pool_metadata_failures_keep_their_typed_recovery_classification() {
use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
+92 -92
View File
@@ -14,7 +14,6 @@
use crate::error::{HealErrorClass, classify_heal_error};
use crate::heal::{
pacing::HealPageRate,
progress::{HealProgress, add_bytes, increment_counter},
resume::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
@@ -129,12 +128,12 @@ pub struct ErasureSetHealer {
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
work_parent: Option<super::manager::HealWork>,
mainline_pacer: Option<super::pacing::ForegroundPacing>,
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
}
async fn acquire_page_permit(
semaphore: Arc<Semaphore>,
rate: &HealPageRate,
pacer: Option<&super::pacing::MainlinePacer>,
cancel: &tokio_util::sync::CancellationToken,
) -> Result<tokio::sync::OwnedSemaphorePermit> {
let acquire = || async {
@@ -147,16 +146,18 @@ async fn acquire_page_permit(
let mut paid_pause = false;
loop {
let permit = acquire().await?;
// Keep the real permit on the low-pressure path. Every acquisition
// gets a fresh decision, including a waiter that queued a second
// time. One completed pause is a bounded minimum-progress grant.
match rate.admission_decision() {
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
drop(permit);
paid_pause = rate.wait_after_admission(cancel, pressure).await?;
continue;
if let Some(pacer) = pacer {
// Keep the real permit on the low-pressure path. Every acquisition
// gets a fresh decision, including a waiter that queued a second
// time. One completed pause is a bounded minimum-progress grant.
match pacer.admission_decision() {
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
drop(permit);
paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
continue;
}
_ => {}
}
_ => {}
}
return Ok(permit);
}
@@ -165,7 +166,7 @@ async fn acquire_page_permit(
#[cfg(test)]
mod mainline_pacing_tests {
use super::*;
use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
use crate::heal::pacing::{MainlinePacer, TestPressure};
use rustfs_concurrency::WorkloadClass;
use std::sync::atomic::Ordering;
use tokio_util::sync::CancellationToken;
@@ -179,15 +180,15 @@ mod mainline_pacing_tests {
.await
.expect("existing object owns capacity");
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
let pacer =
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
let pacer = Arc::new(
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
);
let cancel = CancellationToken::new();
let waiting = tokio::spawn({
let semaphore = semaphore.clone();
let rate = rate.clone();
let pacer = pacer.clone();
let cancel = cancel.clone();
async move { acquire_page_permit(semaphore, &rate, &cancel).await }
async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
});
tokio::task::yield_now().await;
provider.active.store(100, Ordering::SeqCst);
@@ -199,13 +200,12 @@ mod mainline_pacing_tests {
assert_eq!(semaphore.available_permits(), 1);
let deadline = tokio::time::timeout(
Duration::from_millis(10),
acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
)
.await;
assert!(deadline.is_err());
assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
let unpaced = HealPageRate::new(1, None);
let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
.await
.expect("unpaced admission");
assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
@@ -221,11 +221,10 @@ mod mainline_pacing_tests {
let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
let pause = Duration::from_millis(250);
let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
let rate = HealPageRate::new(1, Some(pacer));
let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
let cancel = CancellationToken::new();
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
assert!(futures::poll!(first.as_mut()).is_pending());
assert!(futures::poll!(second.as_mut()).is_pending());
drop(occupied);
@@ -330,20 +329,6 @@ impl ErasureSetHealer {
}
}
/// Page fan-out for one whole erasure-set pass (stable across buckets).
fn pass_page_concurrency_limit(&self) -> usize {
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
// Versions of the same object must not race without its namespace lock.
1
} else {
Self::effective_heal_page_object_concurrency_for_source(
self.source,
self.heal_opts.scan_mode,
self.replacement_task_id.is_some(),
)
}
}
/// Map the shared heal error class onto this healer's per-object actions.
///
/// `classify_heal_error` is the only classifier; this adapter decides what
@@ -388,7 +373,7 @@ impl ErasureSetHealer {
}
}
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
self.mainline_pacer = pacer;
self
}
@@ -479,7 +464,12 @@ impl ErasureSetHealer {
None
};
let buckets = admin_buckets.as_deref().unwrap_or(buckets);
if checkpoint_manager.admin_completed().await {
if checkpoint_manager
.get_checkpoint()
.await
.admin
.is_some_and(|admin| admin.completed)
{
self.initialize_progress(buckets, &resume_manager.get_state().await).await;
return Ok(());
}
@@ -500,7 +490,14 @@ impl ErasureSetHealer {
}
if self.admin_task.is_some() {
checkpoint_manager.complete_admin().await?;
let mut admin = checkpoint_manager
.get_checkpoint()
.await
.admin
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
admin.completed = true;
admin.outcome.finish(None);
checkpoint_manager.set_admin(admin).await?;
} else {
checkpoint_manager.cleanup().await?;
resume_manager.cleanup().await?;
@@ -721,10 +718,7 @@ impl ErasureSetHealer {
}
return Err(Error::DanglingDeleteDeferred { retry_not_before });
}
// One rate controller for the whole pass: SlowDown cool-down and page
// fan-out persist across buckets instead of resetting at each one.
let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
debug!(
target: "rustfs::heal::erasure_healer",
@@ -732,17 +726,14 @@ impl ErasureSetHealer {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
set_disk_id,
current_bucket_index,
current_object_index,
current_bucket_index = checkpoint.current_bucket_index,
current_object_index = checkpoint.current_object_index,
state = "resuming",
"Erasure set resumed"
);
// 2. initialize progress
self.initialize_progress(buckets, &state).await;
// Snapshot before any baseline write: set_progress_baseline overwrites
// the saved generation, which would hide a real generation mismatch.
let checkpoint = checkpoint_manager.progress_snapshot().await;
let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
let baseline = self.progress.read().await;
(
@@ -769,7 +760,17 @@ impl ErasureSetHealer {
let mut successful_objects = state.successful_objects;
let mut failed_objects = state.failed_objects;
let mut skipped_objects = state.skipped_objects;
let checkpoint_has_progress = checkpoint.has_progress();
let checkpoint_has_progress = checkpoint.baseline_known
|| checkpoint.successful_objects > 0
|| checkpoint.failed_object_count > 0
|| checkpoint.skipped_object_count > 0
|| checkpoint.skipped_new_versions > 0
|| checkpoint.skipped_ilm_expired > 0
|| checkpoint.processed_bytes > 0
|| checkpoint.total_objects > 0
|| checkpoint.total_bytes > 0
|| checkpoint.baseline_generation.is_some()
|| checkpoint.counter_unknown;
let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
if checkpoint_has_progress {
@@ -853,7 +854,6 @@ impl ErasureSetHealer {
resume_manager,
checkpoint_manager,
state.start_time,
&page_rate,
)
.await;
@@ -1020,14 +1020,14 @@ impl ErasureSetHealer {
if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
if self.replacement_task_id.is_some() {
let state = resume_manager.get_state().await;
let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
let checkpoint = checkpoint_manager.get_checkpoint().await;
if state.can_retry()
&& failed_objects == 0
&& failed_buckets == 0
&& skipped_objects > 0
&& skipped_objects == grace_objects
&& !counter_unknown
&& let Some(retry_not_before) = retry_not_before
&& skipped_objects == checkpoint.dangling_delete_grace_objects
&& !checkpoint.counter_unknown
&& let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
{
// Publish the generation's rewind and wait first. If the
// checkpoint reset is interrupted, initialization reapplies
@@ -1167,9 +1167,14 @@ impl ErasureSetHealer {
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
let checkpoint_key = compose_key(&object_key, None);
if checkpoint_manager.is_object_settled(&checkpoint_key).await {
let checkpoint = checkpoint_manager.get_checkpoint().await;
if checkpoint.processed_objects.contains(&checkpoint_key)
|| checkpoint.failed_objects.contains(&checkpoint_key)
|| checkpoint.skipped_objects.contains(&checkpoint_key)
{
return Ok(());
}
drop(checkpoint);
self.verify_replacement_identity_fence("pool metadata").await?;
resume_manager
@@ -1423,7 +1428,7 @@ impl ErasureSetHealer {
/// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
async fn heal_bucket_with_resume(
&self,
bucket: &str,
@@ -1437,7 +1442,6 @@ impl ErasureSetHealer {
resume_manager: &ResumeManager,
checkpoint_manager: &CheckpointManager,
started_at_secs: u64,
page_rate: &Arc<HealPageRate>,
) -> Result<bool> {
debug!(
target: "rustfs::heal::erasure_healer",
@@ -1487,6 +1491,17 @@ impl ErasureSetHealer {
// detects a non-advancing cursor even though next_marker embeds a
// transient cache id that changes between identical pages.
let mut previous_page_last: Option<(String, Option<String>)> = None;
let page_concurrency_limit =
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
// Versions of the same object must not race without its namespace lock.
1
} else {
Self::effective_heal_page_object_concurrency_for_source(
self.source,
self.heal_opts.scan_mode,
self.replacement_task_id.is_some(),
)
};
let in_flight = Arc::new(AtomicUsize::new(0));
// Per-bucket sample caps for per-object warn! lines: a flapping rebuild
// disk can fail/skip hundreds of thousands of versions in one sweep, so
@@ -1508,7 +1523,9 @@ impl ErasureSetHealer {
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
loop {
page_rate.wait_before_page(&self.cancel_token).await?;
if let Some(pacer) = &self.mainline_pacer {
pacer.wait(&self.cancel_token).await?;
}
self.verify_replacement_identity_fence("page scan").await?;
if let Some(expected) = incarnation {
self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
@@ -1530,8 +1547,9 @@ impl ErasureSetHealer {
.await?
};
let page_is_empty = objects.is_empty();
let checkpoint = checkpoint_manager.get_checkpoint().await;
let page_resume_index = *current_object_index;
let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
let mut page_tasks = FuturesUnordered::new();
let mut completed_in_page = 0usize;
@@ -1545,7 +1563,10 @@ impl ErasureSetHealer {
// Per-version dedup identity — the single canonical key.
let key = compose_key(&item.name, item.version_id.as_deref());
if checkpoint_manager.is_object_settled(&key).await {
if checkpoint.processed_objects.contains(&key)
|| checkpoint.failed_objects.contains(&key)
|| checkpoint.skipped_objects.contains(&key)
{
continue;
}
@@ -1703,7 +1724,7 @@ impl ErasureSetHealer {
let target_endpoints = self.target_endpoints.clone();
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
let page_rate = page_rate.clone();
let mainline_pacer = self.mainline_pacer.clone();
let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
@@ -1715,15 +1736,14 @@ impl ErasureSetHealer {
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
let work = async move {
let mut retries = 0;
let mut saw_slow_down = false;
let mut admin_identity = admin_identity;
let mut admin_task_id = admin_task_id;
loop {
let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
let permit = match permit {
Ok(permit) => permit,
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
};
let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
@@ -1731,8 +1751,7 @@ impl ErasureSetHealer {
if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
let (result, canonical) =
admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
return (dedup_key, object_name, version_id, result, canonical);
}
// Always go through heal_object. Genuine absence flows through
@@ -1834,23 +1853,18 @@ impl ErasureSetHealer {
drop(in_flight_guard);
drop(permit);
if matches!(&result.1, Err(err) if err.is_slow_down()) {
saw_slow_down = true;
}
if retry_replacement
&& retries < REPLACEMENT_OBJECT_MAX_RETRIES
&& matches!(&result.1, Err(Error::TransientSkip { .. }))
{
retries += 1;
tokio::select! {
_ = cancel_token.cancelled() => {
return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
}
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
_ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
}
continue;
}
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
return (dedup_key, object_name, version_id, result, None);
}
};
page_tasks.push(async move {
@@ -1861,7 +1875,7 @@ impl ErasureSetHealer {
};
match result {
Ok(result) => result,
Err(error) => (key, object, version, (0, Err(error), false), None),
Err(error) => (key, object, version, (0, Err(error)), None),
}
} else {
work.await
@@ -1870,16 +1884,7 @@ impl ErasureSetHealer {
}
while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
let (object_size, result, saw_slow_down) = result;
// Exactly one rate observation per finished object, including
// SlowDown that later succeeded on a replacement retry.
page_rate.observe_object_outcome(
&match &result {
Ok(_) => Ok(true),
Err(err) => Err(err),
},
saw_slow_down,
);
let (object_size, result) = result;
let mut telemetry_unknown = false;
let checkpoint_outcome = match result {
Ok(true) => {
@@ -2477,7 +2482,6 @@ mod resume_loop_tests {
use super::{
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
};
use crate::heal::pacing::HealPageRate;
use crate::heal::progress::HealProgress;
use crate::heal::resume::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
@@ -3059,7 +3063,6 @@ mod resume_loop_tests {
let mut successful = 0u64;
let mut failed = 0u64;
let mut skipped = 0u64;
let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
let result = env
.healer
.heal_bucket_with_resume(
@@ -3074,7 +3077,6 @@ mod resume_loop_tests {
&env.resume,
&env.checkpoint,
state.start_time,
&page_rate,
)
.await;
(processed, successful, failed, skipped, result.map(|_| ()))
@@ -3142,7 +3144,6 @@ mod resume_loop_tests {
let mut skipped = 0;
let started_at = env.resume.get_state().await.start_time;
let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
let error = healer
.heal_bucket_with_resume(
"b",
@@ -3156,7 +3157,6 @@ mod resume_loop_tests {
&env.resume,
&env.checkpoint,
started_at,
&page_rate,
)
.await
.expect_err("a remounted target must not begin a new page scan");
+1 -8
View File
@@ -810,14 +810,7 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
active_heals
.iter()
.find(|(_, task)| {
PriorityHealQueue::make_dedup_key_for_owner(
&task.heal_type,
&task.options,
task.source,
task.expected_mrf_bucket_incarnation_id,
) == key
})
.find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
}
+1 -98
View File
@@ -611,104 +611,7 @@ impl PriorityHealQueue {
/// Create a deduplication key from a heal request
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
Self::make_dedup_key_for_owner(
&request.heal_type,
&request.options,
request.source,
request.expected_mrf_bucket_incarnation_id,
)
}
/// Bound MRF work can only merge with work for the same bucket lifetime.
/// Length-prefix user-controlled components so a key cannot forge another
/// version, scope, or incarnation through delimiter characters.
pub(super) fn make_dedup_key_for_owner(
heal_type: &HealType,
options: &HealOptions,
source: HealRequestSource,
incarnation: Option<Uuid>,
) -> String {
if source == HealRequestSource::Mrf
&& let Some(incarnation) = incarnation
{
use std::fmt::Write as _;
let (pool, set) = heal_scope_indices(heal_type, options);
let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
let mut part = |value: &str| {
// Formatting into a String is infallible.
let _ = write!(key, "{}:{value}", value.len());
};
match heal_type {
HealType::Object {
bucket,
object,
version_id,
} => {
part("object");
part(bucket);
part(object);
match version_id {
None => part("none"),
Some(version_id) => {
part("some");
part(version_id);
}
}
}
HealType::Metadata { bucket, object } => {
part("metadata");
part(bucket);
part(object);
}
HealType::ECDecode {
bucket,
object,
version_id,
} => {
part("ecdecode");
part(bucket);
part(object);
match version_id {
None => part("none"),
Some(version_id) => {
part("some");
part(version_id);
}
}
}
HealType::DeleteMarkerPurge {
bucket,
object,
version_id,
purge,
} => {
part("delete-marker-purge");
part(bucket);
part(object);
part(version_id);
part(&purge.bucket_incarnation_id.to_string());
part(&purge.marker_incarnation_id.to_string());
part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
}
HealType::Bucket { bucket } => {
part("bucket");
part(bucket);
}
HealType::Prefix { bucket, prefix } => {
part("prefix");
part(bucket);
part(prefix);
}
// Cluster and erasure-set work are not bucket-bound; preserve
// their existing keys, including contains_erasure_set lookups.
HealType::Cluster | HealType::ErasureSet { .. } => {
return Self::make_dedup_key_for_scope(heal_type, options);
}
}
return key;
}
Self::make_dedup_key_for_scope(heal_type, options)
Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
}
pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {
+3 -2
View File
@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
request: &HealRequest,
config: &HealConfig,
provider: &Option<WorkloadSnapshotProviderRef>,
) -> Option<crate::heal::pacing::ForegroundPacing> {
) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
let pace_running = request.source == HealRequestSource::Admin
|| (request.source == HealRequestSource::AutoHeal
&& matches!(&request.heal_type, HealType::ErasureSet { .. })
@@ -43,13 +43,14 @@ pub(super) fn running_mainline_pacer(
return None;
}
provider.as_ref().and_then(|provider| {
crate::heal::pacing::ForegroundPacing::new(
crate::heal::pacing::MainlinePacer::new(
provider.clone(),
config.mainline_read_utilization_high_percent,
config.mainline_write_utilization_high_percent,
config.mainline_max_sleep,
request.source,
)
.map(Arc::new)
})
}
-447
View File
@@ -117,453 +117,6 @@ async fn process_manager_queue_once(manager: &HealManager) {
struct MockStorage;
fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
request.source = HealRequestSource::Mrf;
request.expected_mrf_bucket_incarnation_id = Some(incarnation);
request.options.pool_index = Some(0);
request.options.set_index = Some(0);
request
}
async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
let old_incarnation = Uuid::new_v4();
let new_incarnation = Uuid::new_v4();
let old_request = incarnation_bound_mrf_fallback(old_incarnation);
let old_id = old_request.id.clone();
match existing_state {
"queued" => {
let receipt = manager
.submit_heal_request_with_receipt(old_request)
.await
.expect("old fallback must be admitted");
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
}
"active" => {
manager
.active_heals
.lock()
.await
.insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
}
"retrying" => {
manager.retrying_heals.lock().await.insert(
old_id.clone(),
RetryingHeal {
request: old_request,
error: "retry pending".to_owned(),
cancel_token: CancellationToken::new(),
},
);
}
other => panic!("unknown fixture state: {other}"),
}
let new_request = incarnation_bound_mrf_fallback(new_incarnation);
let new_id = new_request.id.clone();
assert_ne!(new_id, old_id, "the fallback must use a new request ID");
let receipt = manager
.submit_heal_request_with_receipt(new_request)
.await
.expect("new incarnation admission must be decided");
assert_eq!(
receipt.result,
HealAdmissionResult::Accepted,
"a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
);
assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
let queue = manager.heal_queue.lock().await;
assert!(
queue
.requests()
.any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
);
if existing_state == "queued" {
assert!(
queue
.requests()
.any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
);
}
}
#[tokio::test]
async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
}
#[tokio::test]
async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
assert_recreated_bucket_fallback_has_its_own_owner("active").await;
}
#[tokio::test]
async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
}
fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
let mut request = incarnation_bound_mrf_fallback(incarnation);
if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
*id = version_id.map(str::to_owned);
} else {
unreachable!("fallback fixture must be an object request");
}
request
}
#[tokio::test]
async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
for version in [Uuid::nil(), Uuid::new_v4()] {
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let incarnation = Uuid::new_v4();
let version_id = version.to_string();
let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
let original_id = original.id.clone();
let accepted = manager
.submit_heal_request_with_receipt(original)
.await
.expect("first queued fallback");
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
let merged = manager
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
.await
.expect("duplicate queued fallback");
assert_eq!(merged.result, HealAdmissionResult::Merged);
assert_eq!(merged.task_id, original_id);
assert_eq!(manager.heal_queue.lock().await.len(), 1);
}
}
#[tokio::test]
async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let incarnation = Uuid::new_v4();
let original = incarnation_bound_mrf_fallback(incarnation);
let original_id = original.id.clone();
assert_eq!(
manager
.submit_heal_request_with_receipt(original)
.await
.expect("first fallback")
.result,
HealAdmissionResult::Accepted
);
let same = manager
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
.await
.expect("same-incarnation fallback");
assert_eq!(same.result, HealAdmissionResult::Merged);
assert_eq!(same.task_id, original_id);
let mut versioned = incarnation_bound_mrf_fallback(incarnation);
versioned.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: "object".to_owned(),
version_id: Some(Uuid::new_v4().to_string()),
};
assert_eq!(
manager
.submit_heal_request_with_receipt(versioned)
.await
.expect("different version")
.result,
HealAdmissionResult::Accepted
);
let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
other_scope.options.pool_index = Some(1);
assert_eq!(
manager
.submit_heal_request_with_receipt(other_scope)
.await
.expect("different pool")
.result,
HealAdmissionResult::Accepted
);
let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
same_id_other_incarnation.id = original_id;
assert_eq!(
manager
.submit_heal_request_with_receipt(same_id_other_incarnation)
.await
.expect("same-ID conflict must be decided")
.result,
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
}
#[test]
fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
let incarnation = Uuid::new_v4();
let bound = incarnation_bound_mrf_fallback(incarnation);
let mut unbound = bound.clone();
unbound.expected_mrf_bucket_incarnation_id = None;
let mut non_mrf = bound.clone();
non_mrf.source = HealRequestSource::ReadRepair;
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
assert_eq!(
PriorityHealQueue::make_dedup_key(&non_mrf),
PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
"an unrelated source retains its previous scope key even if the MRF-only field is set"
);
assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
let mut left = bound.clone();
left.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: "part:a".to_owned(),
version_id: Some("b".to_owned()),
};
let mut right = bound.clone();
right.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: "part".to_owned(),
version_id: Some("a:b".to_owned()),
};
assert_eq!(
PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
"the legacy delimiter-only key demonstrates the collision"
);
assert_ne!(
PriorityHealQueue::make_dedup_key(&left),
PriorityHealQueue::make_dedup_key(&right),
"length-prefixed object and version components must not alias"
);
let mut forged = unbound;
forged.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: PriorityHealQueue::make_dedup_key(&bound),
version_id: None,
};
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
}
#[test]
fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
let incarnation = Uuid::new_v4();
for decode in [false, true] {
let make_request = |object: &str, version_id: Option<&str>| {
let mut request = incarnation_bound_mrf_fallback(incarnation);
let bucket = "mrf-recreated-bucket".to_owned();
let object = object.to_owned();
let version_id = version_id.map(str::to_owned);
request.heal_type = if decode {
HealType::ECDecode {
bucket,
object,
version_id,
}
} else {
HealType::Object {
bucket,
object,
version_id,
}
};
request
};
let absent = make_request("object", None);
let empty = make_request("object", Some(""));
let forged_object = make_request("object4:some0:", None);
let forged_version = make_request("object", Some("4:none"));
let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
for (index, key) in keys.iter().enumerate() {
assert!(keys.iter().skip(index + 1).all(|other| key != other));
}
let mut queue = PriorityHealQueue::new();
assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
assert_eq!(queue.len(), 2);
for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
let mut legacy_absent = absent.clone();
let mut legacy_empty = empty.clone();
legacy_absent.source = source;
legacy_empty.source = source;
if source == HealRequestSource::Mrf {
legacy_absent.expected_mrf_bucket_incarnation_id = None;
legacy_empty.expected_mrf_bucket_incarnation_id = None;
}
assert_eq!(
PriorityHealQueue::make_dedup_key(&legacy_absent),
PriorityHealQueue::make_dedup_key(&legacy_empty),
"unbound and non-MRF work retain the legacy empty-version key"
);
}
}
}
#[test]
fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
let mut queue = PriorityHealQueue::new();
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
let old_id = old.id.clone();
let old_key = PriorityHealQueue::make_dedup_key(&old);
let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
let new_id = new.id.clone();
let new_key = PriorityHealQueue::make_dedup_key(&new);
let mut forced_old = old.clone();
forced_old.id = Uuid::new_v4().to_string();
forced_old.force_start = true;
let forced_id = forced_old.id.clone();
assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
queue
.remove_request_id(&old_id)
.expect("remove the old incarnation representative");
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
}
#[tokio::test]
async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
assert_eq!(
manager
.submit_heal_request_with_receipt(old)
.await
.expect("old fallback")
.result,
HealAdmissionResult::Accepted
);
let incarnation = Uuid::new_v4();
let request = incarnation_bound_mrf_fallback(incarnation);
let request_id = request.id.clone();
let mut intent = MrfIntent {
bucket: Arc::from("mrf-recreated-bucket"),
object: Arc::from("object"),
version_id: None,
kind: MrfKind::PartialWrite,
delete_marker_purge: None,
scope: Some(MrfScope {
pool_index: 0,
set_index: 0,
}),
lease: None,
enqueued_at_ms: 1,
attempts: 0,
};
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
let target = MrfRepairNoticeTarget {
bucket: Arc::clone(&intent.bucket),
object: Arc::clone(&intent.object),
version_id: intent.version_id,
kind: intent.kind,
scope: intent.scope,
delete_marker_purge: None,
lease: intent.lease,
durable_anchor: Some(anchor.clone()),
};
let receipt = manager
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
.await
.expect("durable admission must be decided");
assert_eq!(
receipt.result,
HealAdmissionResult::Accepted,
"a durable new incarnation needs its own owner"
);
assert_eq!(receipt.task_id, request_id);
assert!(
manager
.heal_queue
.lock()
.await
.requests()
.any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
);
assert!(
manager
.try_in_flight_durable_mrf_anchors()
.expect("stable owner snapshot")
.contains(&anchor)
);
}
#[tokio::test]
async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let incarnation = Uuid::new_v4();
let object = format!("same-generation-{}", Uuid::new_v4());
let make_request = || {
let mut request = incarnation_bound_mrf_fallback(incarnation);
request.heal_type = HealType::Object {
bucket: "mrf-recreated-bucket".to_owned(),
object: object.clone(),
version_id: None,
};
request
};
let old = make_request();
let old_id = old.id.clone();
assert_eq!(
manager
.submit_heal_request_with_receipt(old)
.await
.expect("old fallback")
.result,
HealAdmissionResult::Accepted
);
let mut intent = MrfIntent {
bucket: Arc::from("mrf-recreated-bucket"),
object: Arc::from(object.as_str()),
version_id: None,
kind: MrfKind::PartialWrite,
delete_marker_purge: None,
scope: Some(MrfScope {
pool_index: 0,
set_index: 0,
}),
lease: None,
enqueued_at_ms: 1,
attempts: 0,
};
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
let target = MrfRepairNoticeTarget {
bucket: Arc::clone(&intent.bucket),
object: Arc::clone(&intent.object),
version_id: intent.version_id,
kind: intent.kind,
scope: intent.scope,
delete_marker_purge: None,
lease: intent.lease,
durable_anchor: Some(anchor.clone()),
};
let merged = manager
.submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
.await
.expect("durable same-incarnation admission");
assert_eq!(merged.result, HealAdmissionResult::Merged);
assert_eq!(merged.task_id, old_id);
assert!(
manager
.try_in_flight_durable_mrf_anchors()
.expect("stable owner snapshot")
.contains(&anchor)
);
}
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
task_id: "fixture-token".to_string(),
+13 -232
View File
@@ -7,14 +7,8 @@ use rustfs_concurrency::{
workload::{ForegroundPressure, foreground_pressure},
};
use rustfs_heal_contracts::heal_channel::HealRequestSource;
use std::{
sync::{
Arc,
atomic::{AtomicU32, Ordering},
},
time::Duration,
};
use tokio::{sync::Mutex as AsyncMutex, time::Instant};
use std::{sync::Arc, time::Duration};
use tokio::{sync::Mutex, time::Instant};
use tokio_util::sync::CancellationToken;
#[derive(Default)]
@@ -28,19 +22,17 @@ pub(crate) enum PacingDecision {
Wait(Option<ForegroundPressure>),
}
/// Foreground watermarks for one heal execution. Constructed once per request;
/// disabled when watermarks are zero or no workload provider is configured.
#[derive(Clone)]
pub(crate) struct ForegroundPacing {
/// Cooperative pacing for one heal execution, not a storage admission permit.
pub(crate) struct MainlinePacer {
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
read_high: usize,
write_high: usize,
pause: Duration,
source: HealRequestSource,
state: Arc<AsyncMutex<PacingState>>,
state: Mutex<PacingState>,
}
impl ForegroundPacing {
impl MainlinePacer {
pub(crate) fn new(
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
read_high: usize,
@@ -57,7 +49,7 @@ impl ForegroundPacing {
write_high: write_high.min(100),
pause: pause.min(Duration::from_secs(1)),
source,
state: Arc::new(AsyncMutex::new(PacingState::default())),
state: Mutex::new(PacingState::default()),
})
}
@@ -133,142 +125,6 @@ impl ForegroundPacing {
}
}
/// Single owner of "how hard may this heal pass push storage?".
///
/// One instance spans the whole erasure-set scan (all buckets and pages):
/// - foreground admission pressure (`ForegroundPacing`)
/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
///
/// Call `observe_object_outcome` exactly once per finished object so a single
/// SlowDown cannot double-reduce concurrency.
pub(crate) struct HealPageRate {
base_concurrency: usize,
/// 0 = full base concurrency; each SlowDown increases the reduction.
slow_down_step: AtomicU32,
slow_down_until: std::sync::Mutex<Option<Instant>>,
foreground: Option<ForegroundPacing>,
}
impl HealPageRate {
/// Always-on controller. Foreground yield is optional; SlowDown control is not.
pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
Self {
base_concurrency: base_concurrency.max(1),
slow_down_step: AtomicU32::new(0),
slow_down_until: std::sync::Mutex::new(None),
foreground,
}
}
/// Concurrency after applying the current SlowDown reduction. Always ≥ 1
/// so maintenance can still make progress under sustained rate pressure.
pub(crate) fn effective_concurrency(&self) -> usize {
let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
(self.base_concurrency >> step).max(1)
}
/// Fold one finished object into the rate state. Call once per outcome.
/// `saw_slow_down` is true when any attempt of this object hit SlowDown,
/// including attempts that later succeeded on retry.
pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
self.observe_slow_down();
return;
}
if result.is_ok() {
self.observe_success();
}
}
/// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
fn observe_slow_down(&self) {
// fetch_add returns the previous step; +1 so the first SlowDown uses
// exponent 1 (2s), not exponent 0 (1s).
let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
let delay = Self::backoff(step);
let ready = Instant::now() + delay;
if let Ok(mut until) = self.slow_down_until.lock()
&& until.is_none_or(|current| current < ready)
{
*until = Some(ready);
}
}
/// Restore one concurrency step only after the SlowDown cool-down expires.
/// A same-page success must not undo the reduction while the pass is still
/// in the rate-limited window (the page semaphore is fixed until the next page).
fn observe_success(&self) {
let cooling = self
.slow_down_until
.lock()
.ok()
.and_then(|until| *until)
.is_some_and(|ready| ready > Instant::now());
if cooling {
return;
}
self.slow_down_step
.try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
.ok();
}
/// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
self.wait_slow_down(cancel).await?;
if let Some(foreground) = &self.foreground {
foreground.wait(cancel).await?;
}
Ok(())
}
/// Foreground admission decision while the caller owns page capacity.
pub(crate) fn admission_decision(&self) -> PacingDecision {
match &self.foreground {
Some(foreground) => foreground.admission_decision(),
None => PacingDecision::Ready,
}
}
/// Foreground wait after a rejected admission. Returns whether a pause was paid.
pub(crate) async fn wait_after_admission(
&self,
cancel: &CancellationToken,
observed: Option<ForegroundPressure>,
) -> Result<bool> {
match &self.foreground {
Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
None => Ok(false),
}
}
async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
let delay = self
.slow_down_until
.lock()
.ok()
.and_then(|until| *until)
.map(|ready| ready.saturating_duration_since(Instant::now()));
let Some(delay) = delay else {
return Ok(());
};
if delay.is_zero() {
return Ok(());
}
tokio::select! {
biased;
_ = cancel.cancelled() => Err(Error::TaskCancelled),
_ = tokio::time::sleep(delay) => Ok(()),
}
}
fn backoff(step: u32) -> Duration {
// step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
// rate with exponential backoff, and heal must do the same to itself.
let exponent = step.clamp(1, 5);
Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
}
}
#[cfg(test)]
pub(crate) struct TestPressure {
pub(crate) active: std::sync::atomic::AtomicUsize,
@@ -308,87 +164,12 @@ mod tests {
use rustfs_concurrency::WorkloadClass;
use std::sync::atomic::Ordering;
#[tokio::test(start_paused = true)]
async fn heal_page_rate_first_slow_down_waits_two_seconds() {
let rate = HealPageRate::new(8, None);
rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
let cancel = CancellationToken::new();
let before = Instant::now();
rate.wait_before_page(&cancel).await.expect("cool-down wait");
assert_eq!(Instant::now() - before, Duration::from_secs(2));
}
#[tokio::test(start_paused = true)]
async fn heal_page_rate_reduces_concurrency_and_backs_off() {
let rate = HealPageRate::new(8, None);
assert_eq!(rate.effective_concurrency(), 8);
rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
assert_eq!(rate.effective_concurrency(), 4);
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
assert_eq!(rate.effective_concurrency(), 2);
// Floor is one so maintenance never stops entirely.
for _ in 0..8 {
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
}
assert_eq!(rate.effective_concurrency(), 1);
let cancel = CancellationToken::new();
assert!(rate.wait_before_page(&cancel).await.is_ok());
// After cool-down, successes restore concurrency one step at a time.
for _ in 0..12 {
rate.observe_object_outcome(&Ok(true), false);
}
assert!(rate.effective_concurrency() > 1);
}
#[tokio::test(start_paused = true)]
async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
let rate = HealPageRate::new(8, None);
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
assert_eq!(rate.effective_concurrency(), 4);
// Same-page successes while cool-down is armed must not restore fan-out.
for _ in 0..4 {
rate.observe_object_outcome(&Ok(true), false);
}
assert_eq!(rate.effective_concurrency(), 4);
}
#[tokio::test(start_paused = true)]
async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
let rate = HealPageRate::new(8, None);
// Replacement retry: SlowDown on attempt 1, success on attempt 2.
rate.observe_object_outcome(&Ok(true), true);
assert_eq!(rate.effective_concurrency(), 4);
}
#[tokio::test(start_paused = true)]
async fn heal_page_rate_wait_is_cancellable() {
let rate = HealPageRate::new(4, None);
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
let cancel = CancellationToken::new();
cancel.cancel();
assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
}
#[tokio::test(start_paused = true)]
async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
.expect("foreground pacer");
let rate = HealPageRate::new(4, Some(foreground));
let cancel = CancellationToken::new();
let before = Instant::now();
rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
assert_eq!(Instant::now() - before, Duration::from_millis(250));
}
#[tokio::test(start_paused = true)]
async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
let provider = Arc::new(TestPressure::new(class, 0));
let pause = Duration::from_millis(250);
let pacer = ForegroundPacing::new(
let pacer = MainlinePacer::new(
provider.clone(),
if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
@@ -416,9 +197,9 @@ mod tests {
#[tokio::test(start_paused = true)]
async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
let before = Instant::now();
pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
assert_eq!(Instant::now() - before, Duration::from_secs(1));
@@ -428,7 +209,7 @@ mod tests {
async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
let paced = Arc::new(
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
);
let cancel_first = CancellationToken::new();
let first = tokio::spawn({
@@ -447,7 +228,7 @@ mod tests {
cancel_second.cancel();
assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
provider.active.store(0, Ordering::SeqCst);
let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
.expect("independent task");
let before = Instant::now();
other_task
+1 -3
View File
@@ -35,9 +35,7 @@ mod replacement;
mod utils;
pub(crate) use checkpoint::AdminErasureCheckpoint;
pub use checkpoint::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
};
pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
pub(crate) use gc::ResumeGc;
pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};
-105
View File
@@ -152,42 +152,6 @@ impl AdminErasureCheckpoint {
}
}
/// Lightweight progress view for resume decisions. Omits the per-page object
/// ledgers so callers do not clone a million-object checkpoint just to read
/// counters and positions.
#[derive(Debug, Clone, Copy)]
pub struct CheckpointProgressSnapshot {
pub current_bucket_index: usize,
pub current_object_index: usize,
pub successful_objects: u64,
pub failed_object_count: u64,
pub skipped_object_count: u64,
pub skipped_new_versions: u64,
pub skipped_ilm_expired: u64,
pub processed_bytes: u64,
pub total_objects: u64,
pub total_bytes: u64,
pub baseline_generation: Option<u64>,
pub baseline_known: bool,
pub counter_unknown: bool,
}
impl CheckpointProgressSnapshot {
pub fn has_progress(&self) -> bool {
self.baseline_known
|| self.successful_objects > 0
|| self.failed_object_count > 0
|| self.skipped_object_count > 0
|| self.skipped_new_versions > 0
|| self.skipped_ilm_expired > 0
|| self.processed_bytes > 0
|| self.total_objects > 0
|| self.total_bytes > 0
|| self.baseline_generation.is_some()
|| self.counter_unknown
}
}
/// resume checkpoint
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResumeCheckpoint {
@@ -751,75 +715,6 @@ impl CheckpointManager {
self.checkpoint.read().await.clone()
}
/// True when a page object is already settled in the current ledger.
/// Membership is checked under the lock without cloning the checkpoint,
/// which used to copy every page's dedup sets on the small-object path.
pub async fn is_object_settled(&self, key: &str) -> bool {
let checkpoint = self.checkpoint.read().await;
checkpoint.processed_objects.contains(key)
|| checkpoint.failed_objects.contains(key)
|| checkpoint.skipped_objects.contains(key)
}
/// True when the administrator checkpoint already finished.
pub async fn admin_completed(&self) -> bool {
self.checkpoint
.read()
.await
.admin
.as_ref()
.is_some_and(|admin| admin.completed)
}
/// Current resume position without cloning the object ledgers.
pub async fn resume_position(&self) -> (usize, usize) {
let checkpoint = self.checkpoint.read().await;
(checkpoint.current_bucket_index, checkpoint.current_object_index)
}
/// Dangling-delete grace summary without cloning the object ledgers.
pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
let checkpoint = self.checkpoint.read().await;
(
checkpoint.dangling_delete_grace_objects,
checkpoint.dangling_delete_retry_not_before,
checkpoint.counter_unknown,
)
}
/// Aggregate progress counters without cloning the page dedup sets.
pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
let checkpoint = self.checkpoint.read().await;
CheckpointProgressSnapshot {
current_bucket_index: checkpoint.current_bucket_index,
current_object_index: checkpoint.current_object_index,
successful_objects: checkpoint.successful_objects,
failed_object_count: checkpoint.failed_object_count,
skipped_object_count: checkpoint.skipped_object_count,
skipped_new_versions: checkpoint.skipped_new_versions,
skipped_ilm_expired: checkpoint.skipped_ilm_expired,
processed_bytes: checkpoint.processed_bytes,
total_objects: checkpoint.total_objects,
total_bytes: checkpoint.total_bytes,
baseline_generation: checkpoint.baseline_generation,
baseline_known: checkpoint.baseline_known,
counter_unknown: checkpoint.counter_unknown,
}
}
/// Finish the administrator outcome in place and persist it.
pub async fn complete_admin(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
let admin = checkpoint
.admin
.as_mut()
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
admin.completed = true;
admin.outcome.finish(None);
drop(checkpoint);
self.save_checkpoint().await
}
/// update position
pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
+2 -2
View File
@@ -485,7 +485,7 @@ pub struct HealTask {
pub cancel_token: tokio_util::sync::CancellationToken,
/// Storage layer interface
pub storage: Arc<dyn HealStorageAPI>,
mainline_pacer: Option<super::pacing::ForegroundPacing>,
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
}
impl HealTask {
@@ -586,7 +586,7 @@ impl HealTask {
task
}
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
self.mainline_pacer = pacer;
self
}
+2 -3
View File
@@ -480,9 +480,8 @@ impl HealTask {
message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
})?;
if current_bucket_incarnation_id != bucket_incarnation_id {
return Err(Error::StaleBucketIncarnation {
bucket: bucket.to_owned(),
expected: Some(bucket_incarnation_id),
return Err(Error::TaskExecutionFailed {
message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
});
}
expected.bucket_incarnation_id = Some(bucket_incarnation_id);
+1 -58
View File
@@ -3375,67 +3375,10 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
let result = task.heal_object("bucket", "object", None).await;
assert!(matches!(
result,
Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
));
assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
assert!(storage.heal_object_calls.lock().unwrap().is_empty());
}
#[tokio::test]
async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
for object in ["timed out", "network error", "ordinary"] {
let original_incarnation = Uuid::new_v4();
let recreated_incarnation = Uuid::new_v4();
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
..Default::default()
});
let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
request.source = HealRequestSource::Mrf;
request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
request.options.pool_index = Some(0);
request.options.set_index = Some(1);
let task = HealTask::from_request(request, storage.clone());
let error = task
.execute()
.await
.expect_err("a recreated bucket must reject the old MRF responsibility");
assert!(
!error.is_recoverable_heal(),
"user-controlled object names must not make a stale incarnation retryable: {object}"
);
assert!(
matches!(
&error,
Error::StaleBucketIncarnation { bucket, expected }
if bucket == "bucket" && *expected == Some(original_incarnation)
),
"the failure must retain the original bucket incarnation: {error:?}"
);
assert!(
matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
"stale responsibility must be classified independently of the object name"
);
assert!(
storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
"an old responsibility must not reach storage repair in the recreated bucket"
);
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
assert!(
outcome.objects.iter().all(|item| !matches!(
&item.disposition,
HealObjectDisposition::Repaired
| HealObjectDisposition::VerifiedHealthy
| HealObjectDisposition::AuthoritativelyAbsent
)),
"the stale task must not produce a positive repair proof"
);
}
}
#[tokio::test(start_paused = true)]
async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
let storage = Arc::new(MockStorage {
+81 -387
View File
@@ -15,6 +15,7 @@
use std::{
collections::{HashMap, HashSet},
ops::{Deref, DerefMut},
ptr,
sync::{Arc, Mutex, Weak},
};
@@ -63,46 +64,21 @@ impl Default for CacheState {
pub struct Cache {
state: ArcSwap<CacheState>,
write_lock: Mutex<()>,
reload_lock: AsyncMutex<()>,
reload_changes: Mutex<Weak<Mutex<ReloadChanges>>>,
service_account_mutation_lock: AsyncMutex<()>,
sts_account_mutation_locks: Arc<MutationLockRegistry>,
parent_mutation_locks: Arc<MutationLockRegistry>,
sts_account_mutation_locks: Arc<StsMutationLockRegistry>,
}
#[derive(Default)]
struct MutationLockRegistry {
locks: Mutex<HashMap<String, Weak<AsyncMutex<MutationLockState>>>>,
struct StsMutationLockRegistry {
locks: Mutex<HashMap<String, Weak<AsyncMutex<StsMutationLockState>>>>,
}
pub(crate) struct MutationLockState {
pub(crate) struct StsMutationLockState {
access_key: String,
registry: Weak<MutationLockRegistry>,
lock: Weak<AsyncMutex<MutationLockState>>,
registry: Weak<StsMutationLockRegistry>,
lock: Weak<AsyncMutex<StsMutationLockState>>,
}
impl MutationLockRegistry {
fn lock_for(self: &Arc<Self>, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
let mut locks = self.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
return lock;
}
let registry = Arc::downgrade(self);
let access_key_owned = access_key.to_string();
let lock = Arc::new_cyclic(|lock| {
AsyncMutex::new(MutationLockState {
access_key: access_key_owned,
registry,
lock: lock.clone(),
})
});
locks.insert(access_key.to_string(), Arc::downgrade(&lock));
lock
}
}
impl Drop for MutationLockState {
impl Drop for StsMutationLockState {
fn drop(&mut self) {
let Some(registry) = self.registry.upgrade() else {
return;
@@ -119,52 +95,10 @@ impl Default for Cache {
Self {
state: ArcSwap::new(Arc::new(CacheState::default())),
write_lock: Mutex::new(()),
reload_lock: AsyncMutex::new(()),
reload_changes: Mutex::new(Weak::new()),
service_account_mutation_lock: AsyncMutex::new(()),
sts_account_mutation_locks: Arc::new(MutationLockRegistry::default()),
parent_mutation_locks: Arc::new(MutationLockRegistry::default()),
}
}
}
#[derive(Clone, Copy, Hash, PartialEq, Eq)]
pub(crate) enum CachePartition {
PolicyDocs,
Users,
UserPolicies,
StsAccounts,
StsPolicies,
Groups,
UserGroupMemberships,
GroupPolicies,
}
type ReloadChanges = HashMap<CachePartition, HashSet<String>>;
/// A reload journals only keys changed during its own lifetime, including
/// deletions. Unrelated STS traffic cannot suppress a persisted revocation.
pub(crate) struct CacheReload<'a> {
_guard: tokio::sync::MutexGuard<'a, ()>,
changes: Arc<Mutex<ReloadChanges>>,
}
impl CacheReload<'_> {
pub(crate) fn merge_partition<T: Clone>(
&self,
partition: CachePartition,
loaded: &mut CacheEntity<T>,
current: &CacheEntity<T>,
) {
let changes = self.changes.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(keys) = changes.get(&partition) {
for key in keys {
if let Some(value) = current.get(key) {
loaded.insert(key.clone(), value.clone());
} else {
loaded.remove(key);
}
}
sts_account_mutation_locks: Arc::new(StsMutationLockRegistry {
locks: Mutex::new(HashMap::new()),
}),
}
}
}
@@ -172,27 +106,31 @@ impl CacheReload<'_> {
pub(crate) type CacheSnapshot = Guard<Arc<CacheState>>;
impl Cache {
pub(crate) async fn begin_reload(&self) -> CacheReload<'_> {
// Registration and publication both take write_lock before the journal
// lock, so no mutation can fall between the reload and its journal.
let guard = self.reload_lock.lock().await;
let changes = Arc::new(Mutex::new(HashMap::new()));
let _write = self.write_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
*self.reload_changes.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Arc::downgrade(&changes);
CacheReload { _guard: guard, changes }
}
pub(crate) fn service_account_mutation_lock(&self) -> &AsyncMutex<()> {
&self.service_account_mutation_lock
}
pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
self.sts_account_mutation_locks.lock_for(access_key)
}
pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<StsMutationLockState>> {
let mut locks = self
.sts_account_mutation_locks
.locks
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
return lock;
}
/// Orders derived-credential publication with changes to its regular parent.
pub(crate) fn parent_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
self.parent_mutation_locks.lock_for(access_key)
let registry = Arc::downgrade(&self.sts_account_mutation_locks);
let access_key_owned = access_key.to_string();
let lock = Arc::new_cyclic(|lock| {
AsyncMutex::new(StsMutationLockState {
access_key: access_key_owned,
registry,
lock: lock.clone(),
})
});
locks.insert(access_key.to_string(), Arc::downgrade(&lock));
lock
}
pub(crate) fn snapshot(&self) -> CacheSnapshot {
@@ -204,13 +142,7 @@ impl Cache {
let current = self.state.load_full();
let mut locked = LockedCache {
state: CacheState::clone(&current),
#[cfg(test)]
current_ptr: Arc::as_ptr(&current),
reload_changes: self
.reload_changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.upgrade(),
dirty: false,
};
let ret = f(&mut locked);
@@ -283,9 +215,7 @@ impl Cache {
pub(crate) struct LockedCache {
state: CacheState,
#[cfg(test)]
current_ptr: *const CacheState,
reload_changes: Option<Arc<Mutex<ReloadChanges>>>,
dirty: bool,
}
@@ -294,19 +224,11 @@ impl LockedCache {
&self.state
}
#[cfg(test)]
pub(crate) fn matches_snapshot(&self, snapshot: &CacheSnapshot) -> bool {
std::ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
}
fn exec<T: Clone>(
target: &mut Arc<CacheEntity<T>>,
t: OffsetDateTime,
changes: Option<&Arc<Mutex<ReloadChanges>>>,
partition: CachePartition,
key: &str,
mut op: impl FnMut(&mut CacheEntity<T>),
) -> bool {
fn exec<T: Clone>(target: &mut Arc<CacheEntity<T>>, t: OffsetDateTime, mut op: impl FnMut(&mut CacheEntity<T>)) -> bool {
if target.load_time >= t {
return false;
}
@@ -314,14 +236,6 @@ impl LockedCache {
let mut new = CacheEntity::clone(target);
op(&mut new);
*target = Arc::new(new);
if let Some(changes) = changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(partition)
.or_default()
.insert(key.to_string());
}
true
}
@@ -330,315 +244,139 @@ impl LockedCache {
}
pub(crate) fn replace_policy_docs(&mut self, value: CacheEntity<PolicyDoc>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::PolicyDocs)
.or_default()
.extend(self.state.policy_docs.keys().chain(value.keys()).cloned());
}
self.state.policy_docs = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_users(&mut self, value: CacheEntity<UserIdentity>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::Users)
.or_default()
.extend(self.state.users.keys().chain(value.keys()).cloned());
}
self.state.users = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_user_policies(&mut self, value: CacheEntity<MappedPolicy>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::UserPolicies)
.or_default()
.extend(self.state.user_policies.keys().chain(value.keys()).cloned());
}
self.state.user_policies = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_sts_accounts(&mut self, value: CacheEntity<UserIdentity>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::StsAccounts)
.or_default()
.extend(self.state.sts_accounts.keys().chain(value.keys()).cloned());
}
self.state.sts_accounts = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_sts_policies(&mut self, value: CacheEntity<MappedPolicy>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::StsPolicies)
.or_default()
.extend(self.state.sts_policies.keys().chain(value.keys()).cloned());
}
self.state.sts_policies = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_groups(&mut self, value: CacheEntity<GroupInfo>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::Groups)
.or_default()
.extend(self.state.groups.keys().chain(value.keys()).cloned());
}
self.state.groups = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_group_policies(&mut self, value: CacheEntity<MappedPolicy>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::GroupPolicies)
.or_default()
.extend(self.state.group_policies.keys().chain(value.keys()).cloned());
}
self.state.group_policies = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn replace_user_group_memberships(&mut self, value: CacheEntity<HashSet<String>>) {
if let Some(changes) = &self.reload_changes {
changes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.entry(CachePartition::UserGroupMemberships)
.or_default()
.extend(self.state.user_group_memberships.keys().chain(value.keys()).cloned());
}
self.state.user_group_memberships = Self::replaced(value);
self.dirty = true;
}
pub(crate) fn add_or_update_policy_doc(&mut self, key: &str, value: &PolicyDoc, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.policy_docs,
t,
self.reload_changes.as_ref(),
CachePartition::PolicyDocs,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_user(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.users,
t,
self.reload_changes.as_ref(),
CachePartition::Users,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.users, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_user_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.user_policies,
t,
self.reload_changes.as_ref(),
CachePartition::UserPolicies,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_sts_account(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.sts_accounts,
t,
self.reload_changes.as_ref(),
CachePartition::StsAccounts,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_sts_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.sts_policies,
t,
self.reload_changes.as_ref(),
CachePartition::StsPolicies,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_group(&mut self, key: &str, value: &GroupInfo, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.groups,
t,
self.reload_changes.as_ref(),
CachePartition::Groups,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_user_group_membership(&mut self, key: &str, value: &HashSet<String>, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.user_group_memberships,
t,
self.reload_changes.as_ref(),
CachePartition::UserGroupMemberships,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn add_or_update_group_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.group_policies,
t,
self.reload_changes.as_ref(),
CachePartition::GroupPolicies,
key,
|map| {
map.insert(key.to_string(), value.clone());
},
);
self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
map.insert(key.to_string(), value.clone());
});
}
pub(crate) fn delete_policy_doc(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.policy_docs,
t,
self.reload_changes.as_ref(),
CachePartition::PolicyDocs,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_user(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.users,
t,
self.reload_changes.as_ref(),
CachePartition::Users,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.users, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_user_policy(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.user_policies,
t,
self.reload_changes.as_ref(),
CachePartition::UserPolicies,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_user_group_membership(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.user_group_memberships,
t,
self.reload_changes.as_ref(),
CachePartition::UserGroupMemberships,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_sts_account(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.sts_accounts,
t,
self.reload_changes.as_ref(),
CachePartition::StsAccounts,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_sts_policy(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.sts_policies,
t,
self.reload_changes.as_ref(),
CachePartition::StsPolicies,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_group(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.groups,
t,
self.reload_changes.as_ref(),
CachePartition::Groups,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
map.remove(key);
});
}
pub(crate) fn delete_group_policy(&mut self, key: &str, t: OffsetDateTime) {
self.dirty |= Self::exec(
&mut self.state.group_policies,
t,
self.reload_changes.as_ref(),
CachePartition::GroupPolicies,
key,
|map| {
map.remove(key);
},
);
self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
map.remove(key);
});
}
pub(crate) fn build_user_group_memberships(&mut self) {
@@ -756,50 +494,6 @@ mod tests {
use crate::cache::Cache;
use crate::store::MappedPolicy;
#[tokio::test]
async fn reload_journal_preserves_same_key_mutations_and_expires_on_cancellation() {
use super::{CacheEntity, CachePartition};
let cache = Cache::default();
let stale = UserIdentity {
version: 1,
..Default::default()
};
let current = UserIdentity {
version: 2,
..Default::default()
};
let reload = cache.begin_reload().await;
cache.add_or_update_user("updated", &current, OffsetDateTime::now_utc());
cache.add_or_update_user("removed", &current, OffsetDateTime::now_utc());
cache.delete_user("removed", OffsetDateTime::now_utc());
// Even an absent key's deletion is recorded: storage may have returned
// an entry created and deleted entirely during the reload's reads.
cache.delete_user("absent", OffsetDateTime::now_utc());
let mut loaded = CacheEntity::new(std::collections::HashMap::from([
("updated".to_string(), stale.clone()),
("removed".to_string(), stale.clone()),
("absent".to_string(), stale.clone()),
("disk-only".to_string(), stale),
]));
cache.with_write_lock(|locked| {
reload.merge_partition(CachePartition::Users, &mut loaded, &locked.state().users);
locked.replace_users(loaded);
});
let snapshot = cache.snapshot();
assert_eq!(snapshot.users["updated"].version, 2);
assert!(!snapshot.users.contains_key("removed"));
assert!(!snapshot.users.contains_key("absent"));
assert!(snapshot.users.contains_key("disk-only"));
drop(reload);
assert!(cache.reload_changes.lock().expect("journal lock").upgrade().is_none());
let cancelled = cache.begin_reload().await;
cache.delete_user("disk-only", OffsetDateTime::now_utc());
drop(cancelled);
assert!(cache.reload_changes.lock().expect("journal lock").upgrade().is_none());
let next = cache.begin_reload().await;
assert!(next.changes.lock().expect("new journal").is_empty());
}
#[test]
fn sts_mutation_locks_are_keyed_and_prune_unused_entries() {
let cache = Cache::default();
-10
View File
@@ -16,7 +16,6 @@ use super::{FederatedClaims, VerifiedFederatedIdentity};
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
use rustfs_utils::HashAlgorithm;
use serde_json::Value;
use std::{collections::HashMap, sync::Arc};
use tracing::debug;
@@ -67,15 +66,6 @@ impl FederatedAuthorization {
&self.provider_id
}
/// Issuer from the verified upstream identity.
pub fn issuer(&self) -> Option<&str> {
self.claims
.raw
.get("iss")
.and_then(Value::as_str)
.filter(|issuer| !issuer.is_empty())
}
pub fn subject(&self) -> &str {
&self.claims.sub
}
+7 -149
View File
@@ -415,16 +415,6 @@ where
self.api.save_iam_config(IAMFormat::new_version_1(), path).await
}
/// Current built-in IAM group membership, independent of credential snapshots.
pub fn user_group_memberships(&self, name: &str) -> Vec<String> {
self.cache
.snapshot()
.user_group_memberships
.get(name)
.map(|groups| groups.iter().cloned().collect())
.unwrap_or_default()
}
pub async fn get_user(&self, access_key: &str) -> Option<UserIdentity> {
let cache = self.cache.snapshot();
cache
@@ -1015,17 +1005,6 @@ where
}
pub async fn sts_policy_db_get(&self, name: &str, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
self.sts_policy_db_get_with_identity_mapping(name, groups)
.await
.map(|(policies, _)| policies)
}
/// Resolve STS policies and whether that same lookup found a nonempty identity mapping.
pub async fn sts_policy_db_get_with_identity_mapping(
&self,
name: &str,
groups: &Option<Vec<String>>,
) -> Result<(Vec<String>, bool)> {
if name.is_empty() {
return Err(Error::InvalidArgument);
}
@@ -1052,10 +1031,7 @@ where
}
};
let direct_policies = mapped_policy.to_slice();
let has_identity_mapping = !direct_policies.is_empty();
let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
Ok((policies, has_identity_mapping))
self.policy_db_get_with_groups(mapped_policy.to_slice(), groups).await
}
async fn policy_db_get_with_groups(&self, mut policies: Vec<String>, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
@@ -1182,7 +1158,7 @@ where
if let Some(groups) = u.credentials.groups.as_ref() {
for group in groups.iter() {
if groups_cache.get(group).filter(|v| v.status == STATUS_DISABLED).is_some() {
continue;
return Ok((Vec::new(), OffsetDateTime::now_utc()));
}
let mp = match group_policies.get(group) {
@@ -1213,7 +1189,7 @@ where
for group in user_group_memberships.get(name).cloned().unwrap_or_default().iter() {
if groups_cache.get(group).filter(|v| v.status == STATUS_DISABLED).is_some() {
continue;
return Ok((Vec::new(), OffsetDateTime::now_utc()));
}
let mp = match group_policies.get(group) {
@@ -1510,27 +1486,7 @@ where
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
) -> Result<OffsetDateTime> {
self.add_user_at_with_secret_change(access_key, args, updated_at)
.await
.map(|(updated_at, _, _guard)| updated_at)
}
/// Return whether this write replaced a different regular-user secret.
pub(crate) async fn add_user_at_with_secret_change(
&self,
access_key: &str,
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
let parent_lock = self.cache.parent_mutation_lock(access_key);
let parent_guard = parent_lock.lock_owned().await;
let _service_guard = self.cache.service_account_mutation_lock().lock().await;
let sts_lock = self.cache.sts_account_mutation_lock(access_key);
let _sts_guard = sts_lock.lock().await;
let cache = self.cache.snapshot();
if cache.sts_accounts.contains_key(access_key) {
return Err(Error::AccessKeyAlreadyExists);
}
let users = Arc::clone(&cache.users);
if let Some(x) = users.get(access_key) {
debug!(
@@ -1540,13 +1496,10 @@ where
state = "existing",
"Updating existing IAM user"
);
if x.credentials.is_temp() || x.credentials.is_service_account() {
if x.credentials.is_temp() {
return Err(Error::AccessKeyAlreadyExists);
}
}
let secret_changed = users
.get(access_key)
.is_some_and(|user| user.credentials.secret_key != args.secret_key);
drop(cache);
drop(users);
@@ -1570,7 +1523,7 @@ where
self.update_user_with_claims(access_key, user_entry)?;
Ok((updated_at, secret_changed, parent_guard))
Ok(updated_at)
}
pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> {
@@ -1587,17 +1540,12 @@ where
return Err(Error::InvalidArgument);
}
let parent_lock = (utype == UserType::Reg).then(|| self.cache.parent_mutation_lock(access_key));
let _parent_guard = match &parent_lock {
Some(lock) => Some(lock.lock().await),
None => None,
};
let sts_mutation_lock = (utype == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(access_key));
let _sts_mutation_guard = match &sts_mutation_lock {
Some(lock) => Some(lock.lock().await),
None => None,
};
let _service_account_guard = if matches!(utype, UserType::Svc | UserType::Reg) {
let _service_account_guard = if utype == UserType::Svc {
Some(self.cache.service_account_mutation_lock().lock().await)
} else {
None
@@ -1692,46 +1640,17 @@ where
}
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
self.update_user_secret_key_with_guard(access_key, secret_key, None)
.await
.map(|(updated_at, status, _guard)| (updated_at, status))
}
pub(crate) async fn update_user_secret_key_with_guard(
&self,
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
) -> Result<(
OffsetDateTime,
AccountStatus,
tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>,
)> {
if access_key.is_empty() || secret_key.is_empty() {
return Err(Error::InvalidArgument);
}
let parent_lock = self.cache.parent_mutation_lock(access_key);
let parent_guard = parent_lock.lock_owned().await;
let cache = self.cache.snapshot();
let users = Arc::clone(&cache.users);
let u = match users.get(access_key) {
Some(u) => u,
None if expected_secret.is_some() => return Err(Error::IAMActionNotAllowed),
None => return Err(Error::NoSuchUser(access_key.to_string())),
};
if let Some(expected) = expected_secret {
use subtle::ConstantTimeEq as _;
if !u.credentials.is_valid()
|| u.credentials.is_temp()
|| u.credentials.is_service_account()
|| !bool::from(u.credentials.secret_key.as_bytes().ct_eq(expected.as_bytes()))
{
return Err(Error::IAMActionNotAllowed);
}
}
let mut cred = u.credentials.clone();
cred.secret_key = secret_key.to_string();
@@ -1753,7 +1672,7 @@ where
.await?;
self.update_user_with_claims(access_key, u)?;
Ok((updated_at, status, parent_guard))
Ok((updated_at, status))
}
/// Add SSH public key for a user (for SFTP authentication)
@@ -1762,8 +1681,6 @@ where
return Err(Error::InvalidArgument);
}
let parent_lock = self.cache.parent_mutation_lock(access_key);
let _parent_guard = parent_lock.lock().await;
let cache = self.cache.snapshot();
let users = Arc::clone(&cache.users);
let u = match users.get(access_key) {
@@ -1803,8 +1720,6 @@ where
return Err(Error::InvalidArgument);
}
let parent_lock = self.cache.parent_mutation_lock(access_key);
let _parent_guard = parent_lock.lock().await;
let cache = self.cache.snapshot();
let users = Arc::clone(&cache.users);
let u = match users.get(access_key) {
@@ -2323,11 +2238,6 @@ where
Ok(())
}
pub async fn user_notification_handler(&self, name: &str, user_type: UserType) -> Result<()> {
let parent_lock = (user_type == UserType::Reg).then(|| self.cache.parent_mutation_lock(name));
let _parent_guard = match &parent_lock {
Some(lock) => Some(lock.lock().await),
None => None,
};
let sts_mutation_lock = (user_type == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(name));
let _sts_mutation_guard = match &sts_mutation_lock {
Some(lock) => Some(lock.lock().await),
@@ -3075,58 +2985,6 @@ mod tests {
}
}
#[tokio::test]
async fn disabled_group_preserves_direct_and_other_group_policies() {
for credential_groups in [false, true] {
let iam = build_test_iam_cache(FailingInitialLoadStore);
let groups = vec!["disabled".to_string(), "enabled".to_string()];
let user = UserIdentity::new(Credentials {
access_key: "alice".to_string(),
secret_key: "alice-secret".to_string(),
status: STATUS_ENABLED.to_string(),
groups: credential_groups.then(|| groups.clone()),
..Default::default()
});
let now = OffsetDateTime::now_utc();
iam.cache.add_or_update_user("alice", &user, now);
iam.cache
.add_or_update_user_policy("alice", &MappedPolicy::new("direct-deny"), now);
if !credential_groups {
iam.cache
.add_or_update_user_group_membership("alice", &groups.iter().cloned().collect(), now);
}
for (group, status, policy) in [
("disabled", STATUS_DISABLED, "disabled-grant"),
("enabled", STATUS_ENABLED, "other-group-deny"),
] {
iam.cache.add_or_update_group(
group,
&GroupInfo {
status: status.to_string(),
members: vec!["alice".to_string()],
..Default::default()
},
now,
);
iam.cache.add_or_update_group_policy(group, &MappedPolicy::new(policy), now);
}
let policies: HashSet<_> = iam
.policy_db_get("alice", &None)
.await
.expect("user policy lookup")
.into_iter()
.collect();
assert_eq!(policies, HashSet::from(["direct-deny".to_string(), "other-group-deny".to_string()]));
assert!(
iam.policy_db_get_internal("disabled", true, false)
.await
.expect("disabled group lookup")
.0
.is_empty()
);
}
}
#[tokio::test]
async fn sts_policy_lookup_ignores_colliding_regular_user_mapping() {
let iam = build_test_iam_cache(FailingInitialLoadStore);
+15 -90
View File
@@ -16,21 +16,19 @@ use super::{
transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
};
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
use openidconnect::{
AsyncHttpClient, ClaimsVerificationError, ClientId, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
};
use serde::Deserialize;
use std::collections::HashMap;
use std::sync::{Arc, Mutex, RwLock};
use std::sync::RwLock;
use std::time::{Duration as StdDuration, Instant};
use tokio::time::sleep;
use tracing::{debug, warn};
use url::Url;
pub(super) const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
const OIDC_JWKS_REFRESH_COOLDOWN: StdDuration = StdDuration::from_secs(60);
pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
pub(super) const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy";
@@ -98,14 +96,9 @@ pub(super) struct ProviderState {
pub(super) discovered_at: Instant,
}
type RefreshAttempt = Option<(Instant, Result<ProviderState, String>)>;
type ProviderRefreshLock = Arc<tokio::sync::Mutex<RefreshAttempt>>;
#[derive(Clone)]
pub(super) struct ProviderRuntime {
states: Arc<RwLock<HashMap<String, ProviderState>>>,
http_client: Arc<ReqwestHttpClient>,
refreshes: Arc<Mutex<HashMap<String, ProviderRefreshLock>>>,
states: RwLock<HashMap<String, ProviderState>>,
http_client: ReqwestHttpClient,
}
// Workload issuers do not implement the browser authorization flow. Keep their
@@ -206,57 +199,19 @@ impl DiscoveredProviderMetadata {
pub(super) fn verifier(&self, config: &OidcProviderConfig) -> CoreIdTokenVerifier<'static> {
let client_id = ClientId::new(config.client_id.clone());
let secret = config.client_secret.as_ref().map(|secret| ClientSecret::new(secret.clone()));
let (issuer, jwks, algorithms) = match self {
Self::Console(metadata) => (metadata.issuer(), metadata.jwks(), metadata.id_token_signing_alg_values_supported()),
Self::Workload(metadata) => (&metadata.issuer, &metadata.jwks, &metadata.id_token_signing_alg_values_supported),
};
// Caller-supplied tokens must prove possession of the issuer's signing
// key. A client secret is shared with clients and cannot provide that proof.
let algorithms = algorithms
.iter()
.filter(|algorithm| {
!matches!(
algorithm,
CoreJwsSigningAlgorithm::HmacSha256
| CoreJwsSigningAlgorithm::HmacSha384
| CoreJwsSigningAlgorithm::HmacSha512
)
})
.cloned()
.collect::<Vec<_>>();
CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()).set_allowed_algs(algorithms)
let verifier = match secret {
Some(secret) => CoreIdTokenVerifier::new_confidential_client(client_id, secret, issuer.clone(), jwks.clone()),
None => CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()),
};
verifier.set_allowed_algs(algorithms.clone())
}
}
pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
token: &'a CoreIdToken,
verifier: &CoreIdTokenVerifier<'_>,
nonce_verifier: N,
) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
CoreJwsSigningAlgorithm::HmacSha256 => 32,
CoreJwsSigningAlgorithm::HmacSha384 => 48,
CoreJwsSigningAlgorithm::HmacSha512 => 64,
_ => return token.claims(verifier, nonce_verifier),
};
// RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
// openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
let invalid_length = || {
ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
"invalid HMAC signature length".to_owned(),
))
};
let compact = token.to_string();
let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
let signature = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(signature)
.map_err(|_| invalid_length())?;
if signature.len() != expected_length {
return Err(invalid_length());
}
token.claims(verifier, nonce_verifier)
}
// This adapter is used only for discovery/JWKS fetches, never token exchange.
struct JwksAcceptClient<'a> {
inner: &'a ReqwestHttpClient,
@@ -288,9 +243,8 @@ impl ProviderState {
impl ProviderRuntime {
pub(super) fn new(http_client: ReqwestHttpClient, states: HashMap<String, ProviderState>) -> Self {
Self {
states: Arc::new(RwLock::new(states)),
http_client: Arc::new(http_client),
refreshes: Arc::new(Mutex::new(HashMap::new())),
states: RwLock::new(states),
http_client,
}
}
@@ -342,38 +296,9 @@ impl ProviderRuntime {
provider_id: &str,
config: &OidcProviderConfig,
) -> Result<ProviderState, String> {
let refresh = self
.refreshes
.lock()
.map_err(|_| "provider refresh registry is unavailable".to_string())?
.entry(provider_id.to_string())
.or_default()
.clone();
let mut attempt = refresh.lock_owned().await;
if let Some((completed, result)) = attempt.as_ref()
&& completed.elapsed() < OIDC_JWKS_REFRESH_COOLDOWN
{
return result.clone();
}
// The admitted fetch owns the per-provider lock. A disconnected request
// cannot abandon discovery while consuming the provider's cooldown.
// Waiting callers share its completed result, including genuine failures.
let runtime = tokio::runtime::Handle::try_current().map_err(|err| err.to_string())?;
let provider_runtime = self.clone();
let provider_id = provider_id.to_string();
let config = config.clone();
runtime
.spawn(async move {
let result = provider_runtime
.recover_provider_state(&provider_id, &config)
.await
.map_err(|err| err.to_string());
*attempt = Some((Instant::now(), result.clone()));
result
})
self.recover_provider_state(provider_id, config)
.await
.map_err(|err| err.to_string())?
.map_err(|err| err.to_string())
}
pub(super) async fn recover_provider_state(
+22 -173
View File
@@ -180,7 +180,7 @@ fn workload_metadata_rejects_duplicate_fields() {
}
}
#[test]
fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
fn workload_verifier_preserves_algorithm_secret_and_audience_policy() {
let secret = Nonce::new_random().secret().to_string();
let mut config = build_mocked_oidc_provider_config("workload", "https://issuer.example.com");
config.client_secret = Some(secret.clone());
@@ -195,8 +195,7 @@ fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
let token: CoreIdToken = signed.parse().unwrap();
for workload in [false, true] {
for (algorithms, accepted) in [
(serde_json::json!(["RS256", "HS256"]), false),
(serde_json::json!(["HS256", "unsupported-future-algorithm"]), false),
(serde_json::json!(["HS256", "unsupported-future-algorithm"]), true),
(serde_json::json!(["ES256"]), false),
(serde_json::json!(["unsupported-future-algorithm"]), false),
] {
@@ -217,6 +216,21 @@ fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
accepted,
"workload={workload}, algorithms={algorithms}"
);
if accepted {
assert!(
token.claims(&metadata.verifier(&config), |_: Option<&Nonce>| Ok(())).is_err(),
"additional audiences require explicit trust"
);
let mut wrong_secret = config.clone();
wrong_secret.client_secret = Some(Nonce::new_random().secret().to_string());
let verifier = metadata
.verifier(&wrong_secret)
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
assert!(
token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
"incorrect client secret must fail"
);
}
}
}
}
@@ -228,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
let now = time::OffsetDateTime::now_utc().unix_timestamp();
let payload = serde_json::json!({
"iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
"iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
"iat": now, "exp": now + 300,
});
for (algorithm, name, signature_length) in [
@@ -261,41 +275,22 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
.expect("parse configured provider metadata");
let verifier = metadata.verifier(&config);
assert!(
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
"caller-supplied HMAC tokens must be rejected"
);
// The authorization-code flow obtains tokens directly from the
// provider and still supports confidential-client verification.
let verifier = openidconnect::core::CoreIdTokenVerifier::new_confidential_client(
openidconnect::ClientId::new(config.client_id.clone()),
openidconnect::ClientSecret::new(secret.to_owned()),
metadata.issuer().clone(),
openidconnect::core::CoreJsonWebKeySet::new(vec![]),
)
.set_allowed_algs(vec![serde_json::from_value(serde_json::json!(name)).expect("parse signing algorithm")]);
assert!(
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
"valid {name} token must pass for workload={workload}"
);
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
let malformed: CoreIdToken = format!("{signing_input}.{signature}")
.parse()
.expect("invalid signature length must remain parseable before verification");
.expect("invalid signature length must still reach cryptographic verification");
assert!(
verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
"{name} signature length {length} must return an error for workload={workload}"
);
assert!(
verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
"{name} signature length {length} must return an error with a nonce for workload={workload}"
);
}
assert!(
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
"invalid inputs must not prevent later valid {name} verification"
);
}
@@ -779,149 +774,3 @@ async fn test_validate_oidc_provider_config_returns_detailed_errors() {
assert!(err.contains(&format!("issuer '{base}/application/o/rustfs/'")));
assert!(handle.join().is_ok());
}
#[tokio::test]
async fn jwks_refresh_is_single_flight_and_survives_request_cancellation() {
let (initial_key, initial_jwk) = oidc_es256_key_and_jwk("initial");
let (rotated_key, rotated_jwk) = oidc_es256_key_and_jwk("rotated");
let (wrong_key, _) = oidc_es256_key_and_jwk("wrong");
let requests = Arc::new(AtomicUsize::new(0));
let started = Arc::new(tokio::sync::Notify::new());
let (release_tx, release_rx) = std::sync::mpsc::channel();
let server_requests = Arc::clone(&requests);
let server_started = Arc::clone(&started);
let Some((base, server)) = start_mock_oidc_discovery_server_with_jwks(
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
6,
"ES256",
false,
move |fetch| {
let request = server_requests.fetch_add(1, Ordering::SeqCst);
if request == 2 {
server_started.notify_one();
release_rx
.recv_timeout(StdDuration::from_secs(5))
.expect("release refresh response");
}
serde_json::json!({"keys": [if fetch == 0 { &initial_jwk } else { &rotated_jwk }]}).to_string()
},
) else {
return;
};
let mut config = build_mocked_oidc_provider_config("bounded-refresh", &base);
config.issuer = Some(base.clone());
let http = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).expect("allow test origin"));
let state = discover_provider(&config, &http).await.expect("initial discovery");
let sys = Arc::new(OidcSys {
configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]),
provider_runtime: ProviderRuntime::new(http, HashMap::from([(config.id.clone(), state)])),
state_store: OidcStateStore::new(),
pending_discovery: Default::default(),
});
let now = time::OffsetDateTime::now_utc().unix_timestamp();
let payload = serde_json::json!({"iss": base, "sub": "user", "aud": config.client_id, "iat": now, "exp": now + 300});
let sign = |kid: &str, claims: &serde_json::Value, key: &EncodingKey| {
let mut header = Header::new(Algorithm::ES256);
header.kid = Some(kid.to_string());
jsonwebtoken::encode(&header, claims, key).expect("sign test token")
};
for (claim, value) in [
("aud", serde_json::json!("wrong-audience")),
("exp", serde_json::json!(now - 300)),
] {
let mut invalid = payload.clone();
invalid[claim] = value;
assert!(
sys.verify_web_identity_token(&sign("initial", &invalid, &initial_key))
.await
.is_err()
);
}
assert_eq!(requests.load(Ordering::SeqCst), 2, "claim failures cannot refresh signing keys");
let invalid = sign("rotated", &payload, &wrong_key);
let attack = {
let sys = Arc::clone(&sys);
let invalid = invalid.clone();
tokio::spawn(async move { sys.verify_web_identity_token(&invalid).await })
};
tokio::time::timeout(StdDuration::from_secs(5), started.notified())
.await
.expect("refresh started");
attack.abort();
assert!(attack.await.expect_err("request canceled").is_cancelled());
let mut retries = Vec::new();
for _ in 0..16 {
let sys = Arc::clone(&sys);
let invalid = invalid.clone();
retries.push(tokio::spawn(async move { sys.verify_web_identity_token(&invalid).await }));
}
release_tx.send(()).expect("release provider");
for retry in retries {
assert!(
tokio::time::timeout(StdDuration::from_secs(5), retry)
.await
.expect("retry timeout")
.expect("retry join")
.is_err()
);
}
let rotated = sign("rotated", &payload, &rotated_key);
sys.verify_web_identity_token(&rotated)
.await
.expect("completed refresh must accept rotated keys after the original caller left");
assert_eq!(requests.load(Ordering::SeqCst), 4, "one discovery/JWKS pair for all rejected tokens");
sys.provider_runtime
.refresh_provider_state("independent-provider", &config)
.await
.expect("provider cooldowns must be independent");
assert_eq!(requests.load(Ordering::SeqCst), 6);
server.join().expect("mock server");
}
#[tokio::test]
async fn jwks_refresh_cools_down_failures_and_recovers_after_the_interval() {
let (_, initial_jwk) = oidc_es256_key_and_jwk("initial");
let (_, recovered_jwk) = oidc_es256_key_and_jwk("recovered");
let requests = Arc::new(AtomicUsize::new(0));
let server_requests = Arc::clone(&requests);
let Some((base, server)) = start_mock_oidc_discovery_server_with_jwks(
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
6,
"ES256",
false,
move |fetch| {
server_requests.fetch_add(1, Ordering::SeqCst);
match fetch {
0 => serde_json::json!({"keys": [initial_jwk]}).to_string(),
1 => "invalid JWKS".to_string(),
_ => serde_json::json!({"keys": [recovered_jwk]}).to_string(),
}
},
) else {
return;
};
let mut config = build_mocked_oidc_provider_config("recovering-refresh", &base);
config.issuer = Some(base.clone());
let http = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).expect("allow test origin"));
let initial = discover_provider(&config, &http).await.expect("initial discovery");
let runtime = ProviderRuntime::new(http, HashMap::from([(config.id.clone(), initial.clone())]));
assert!(runtime.refresh_provider_state(&config.id, &config).await.is_err());
assert_eq!(requests.load(Ordering::SeqCst), 4);
for _ in 0..8 {
assert!(runtime.refresh_provider_state(&config.id, &config).await.is_err());
}
assert_eq!(requests.load(Ordering::SeqCst), 4, "completed failures also consume a bounded cooldown");
assert_eq!(
runtime.get_provider_state(&config.id).expect("cached keys").discovered_at,
initial.discovered_at
);
let refresh = runtime.refreshes.lock().expect("registry")[&config.id].clone();
refresh.lock().await.as_mut().expect("completed attempt").0 = Instant::now() - OIDC_JWKS_REFRESH_COOLDOWN;
runtime
.refresh_provider_state(&config.id, &config)
.await
.expect("retry after cooldown");
assert_eq!(requests.load(Ordering::SeqCst), 6);
assert!(runtime.get_provider_state(&config.id).expect("recovered keys").discovered_at > initial.discovered_at);
server.join().expect("mock server");
}
+9 -13
View File
@@ -16,7 +16,7 @@ use super::{
OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
SourcedOidcProviderConfig, load_oidc_config_snapshot,
},
provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
provider::{ProviderRuntime, discover_provider_for_recovery},
state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
};
@@ -33,8 +33,8 @@ use crate::{
};
use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken};
use openidconnect::{
Audience, AuthType, AuthorizationCode, ClaimsVerificationError, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce,
PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope,
Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce, PkceCodeChallenge,
PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope,
};
use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
use rustfs_utils::logging::LogThrottle;
@@ -535,11 +535,8 @@ impl OidcSys {
let verifier = client
.id_token_verifier()
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
if let Err(e) = verified {
if !matches!(e, ClaimsVerificationError::SignatureVerification(_)) {
return Err(format!("ID token verification failed: {e}"));
}
let verification_error = e.to_string();
warn!(
event = EVENT_OIDC_DIAGNOSTICS,
@@ -586,7 +583,8 @@ impl OidcSys {
let verifier = client
.id_token_verifier()
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
id_token
.claims(&verifier, &Nonce::new(session.nonce.clone()))
.map_err(|retry_err| {
error!(
event = EVENT_OIDC_DIAGNOSTICS,
@@ -769,10 +767,7 @@ impl OidcSys {
.metadata
.verifier(&config)
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
if !matches!(e, ClaimsVerificationError::SignatureVerification(_)) {
return Err(format!("ID token verification failed: {e}"));
}
if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
state = self
.provider_runtime
.refresh_provider_state(&provider_id, &config)
@@ -785,7 +780,8 @@ impl OidcSys {
.metadata
.verifier(&config)
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
id_token
.claims(&verifier, |_: Option<&Nonce>| Ok(()))
.map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
}
-14
View File
@@ -14,20 +14,6 @@
use std::sync::Arc;
pub(crate) fn is_unreadable_bucket_tags(error: &crate::error::Error) -> bool {
let crate::error::Error::Io(error) = error else {
return false;
};
let Some(storage_error) = error.get_ref().and_then(|inner| inner.downcast_ref::<IamStorageError>()) else {
return false;
};
rustfs_ecstore::api::bucket::metadata::unreadable_config_refusal(storage_error)
.is_some_and(|failure| failure.config_file == rustfs_ecstore::api::bucket::metadata::BUCKET_TAGGING_CONFIG)
}
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata::UnreadableBucketConfig;
pub(crate) use rustfs_ecstore::api::config::RUSTFS_CONFIG_PREFIX as IAM_CONFIG_ROOT_PREFIX;
use rustfs_ecstore::api::config::com::{
delete_config as ecstore_delete_config, read_config_no_lock as ecstore_read_config_no_lock,
+37 -146
View File
@@ -1154,7 +1154,7 @@ impl Store for ObjectStore {
// node-counted lock quorum (rustfs#4304; startup contract rustfs#4056).
const LOAD_ALL_MODE: LoadMode = LoadMode::BootstrapNoLock;
let reload = cache.begin_reload().await;
let cache_snapshot = cache.snapshot();
let listed_config_items = self.list_all_iamconfig_items().await?;
let mut policy_docs_cache = CacheEntity::new(get_default_policies());
@@ -1201,20 +1201,26 @@ impl Store for ObjectStore {
}
// groups
let mut groups_cache = CacheEntity::default();
let mut groups_cache = None;
if let Some(item_name_list) = listed_config_items.get(GROUPS_LIST_KEY) {
let mut items_cache = CacheEntity::default();
for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item);
debug!(group = %name, "IAM group loaded");
if let Err(err) = self.load_group_with(&name, &mut groups_cache, LOAD_ALL_MODE).await {
if let Err(err) = self.load_group_with(&name, &mut items_cache, LOAD_ALL_MODE).await {
return Err(Error::other(format!("load group failed: {err}")));
};
}
groups_cache = Some(items_cache);
}
// user policies
let mut user_policies_cache = CacheEntity::default();
let mut user_policies_cache = None;
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_USERS_LIST_KEY) {
let mut items_cache = CacheEntity::default();
// Load in fixed-size chunks so each mapping is fetched exactly once
// (backlog#806 — the split_off loop was O(n^2)).
for chunk in item_name_list.chunks(32) {
@@ -1229,26 +1235,32 @@ impl Store for ObjectStore {
let name = chunk[idx].trim_end_matches(".json").to_owned();
debug!(user = %name, "IAM user policy loaded");
user_policies_cache.insert(name, p);
items_cache.insert(name, p);
}
}
user_policies_cache = Some(items_cache);
}
// group policy
let mut group_policies_cache = CacheEntity::default();
let mut group_policies_cache = None;
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_GROUPS_LIST_KEY) {
let mut items_cache = CacheEntity::default();
for item in item_name_list.iter() {
let name = item.trim_end_matches(".json");
debug!(group = %name, "IAM group policy loaded");
if let Err(err) = self
.load_mapped_policy_with(name, UserType::Reg, true, &mut group_policies_cache, LOAD_ALL_MODE)
.load_mapped_policy_with(name, UserType::Reg, true, &mut items_cache, LOAD_ALL_MODE)
.await
&& !is_err_no_such_policy(&err)
{
return Err(Error::other(format!("load group policy failed: {err}")));
};
}
group_policies_cache = Some(items_cache);
}
let mut sts_policies_cache = CacheEntity::default();
@@ -1318,48 +1330,25 @@ impl Store for ObjectStore {
}
}
#[cfg(test)]
if let Ok(barrier) = LOAD_ALL_COMMIT_BARRIER.try_with(std::sync::Arc::clone) {
barrier.0.notify_one();
barrier.1.notified().await;
}
cache.with_write_lock(|cache| {
reload.merge_partition(
crate::cache::CachePartition::PolicyDocs,
&mut policy_docs_cache,
&cache.state().policy_docs,
);
reload.merge_partition(crate::cache::CachePartition::Groups, &mut groups_cache, &cache.state().groups);
reload.merge_partition(
crate::cache::CachePartition::UserPolicies,
&mut user_policies_cache,
&cache.state().user_policies,
);
reload.merge_partition(
crate::cache::CachePartition::GroupPolicies,
&mut group_policies_cache,
&cache.state().group_policies,
);
reload.merge_partition(crate::cache::CachePartition::Users, &mut user_items_cache, &cache.state().users);
reload.merge_partition(
crate::cache::CachePartition::StsAccounts,
&mut sts_items_cache,
&cache.state().sts_accounts,
);
reload.merge_partition(
crate::cache::CachePartition::StsPolicies,
&mut sts_policies_cache,
&cache.state().sts_policies,
);
cache.replace_policy_docs(policy_docs_cache);
cache.replace_groups(groups_cache);
cache.replace_user_policies(user_policies_cache);
cache.replace_group_policies(group_policies_cache);
cache.replace_users(user_items_cache);
cache.replace_sts_accounts(sts_items_cache);
cache.replace_sts_policies(sts_policies_cache);
cache.build_user_group_memberships();
if cache.matches_snapshot(&cache_snapshot) {
cache.replace_policy_docs(policy_docs_cache);
if let Some(groups_cache) = groups_cache {
cache.replace_groups(groups_cache);
}
if let Some(user_policies_cache) = user_policies_cache {
cache.replace_user_policies(user_policies_cache);
}
if let Some(group_policies_cache) = group_policies_cache {
cache.replace_group_policies(group_policies_cache);
}
cache.replace_users(user_items_cache);
cache.replace_sts_accounts(sts_items_cache);
cache.replace_sts_policies(sts_policies_cache);
cache.build_user_group_memberships();
} else {
warn!("IAM full reload cache commit skipped due to concurrent cache changes");
}
});
Ok(())
@@ -1374,11 +1363,6 @@ fn map_delete_user_identity_error(name: &str, err: Error) -> Error {
}
}
#[cfg(test)]
tokio::task_local! {
static LOAD_ALL_COMMIT_BARRIER: std::sync::Arc<(tokio::sync::Notify, tokio::sync::Notify)>;
}
#[cfg(test)]
mod tests {
use super::{DecryptSource, LoadMode, ObjectStore, map_delete_user_identity_error};
@@ -1388,99 +1372,6 @@ mod tests {
use serial_test::serial;
use temp_env::with_vars;
#[test]
#[serial]
fn test_load_all_clears_partitions_with_empty_prefix() {
use crate::cache::Cache;
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
std::thread::Builder::new()
.stack_size(8 * 1024 * 1024)
.spawn(|| {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.expect("build IAM reload test runtime");
runtime.block_on(async {
let _credentials = test_cred();
let temp = tempfile::tempdir().expect("create IAM reload store directory");
let env = Box::pin(
rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(temp.path())
.init_bucket_metadata(false)
.build(),
)
.await;
let store = ObjectStore::new(env.ecstore);
store
.save_group_info("reload-group", GroupInfo::new(vec!["reload-user".to_string()]))
.await
.expect("seed group membership");
store
.save_mapped_policy("reload-user", UserType::Reg, false, MappedPolicy::new("readwrite"), None)
.await
.expect("seed user policy");
store
.save_mapped_policy("reload-group", UserType::Reg, true, MappedPolicy::new("readonly"), None)
.await
.expect("seed group policy");
let cache = Cache::default();
store.load_all(&cache).await.expect("load nonempty IAM partitions");
{
let snapshot = cache.snapshot();
assert!(snapshot.groups.contains_key("reload-group"));
assert_eq!(snapshot.user_policies["reload-user"].policies, "readwrite");
assert_eq!(snapshot.group_policies["reload-group"].policies, "readonly");
assert!(snapshot.user_group_memberships["reload-user"].contains("reload-group"));
}
// Mutate durable state without notifying this cache, as on
// a peer that missed the last mapping/group deletion.
store
.delete_mapped_policy("reload-user", UserType::Reg, false)
.await
.expect("remove last user mapping");
store
.delete_mapped_policy("reload-group", UserType::Reg, true)
.await
.expect("remove last group mapping");
store.delete_group_info("reload-group").await.expect("remove last group");
assert!(cache.snapshot().groups.contains_key("reload-group"));
// Simulate a missed peer revocation while STS publication
// and a separate user update overlap this full disk reload.
let stale = rustfs_policy::auth::UserIdentity {
version: 1,
..Default::default()
};
cache.add_or_update_user("revoked-user", &stale, time::OffsetDateTime::now_utc());
let barrier = std::sync::Arc::new((tokio::sync::Notify::new(), tokio::sync::Notify::new()));
let loading = super::LOAD_ALL_COMMIT_BARRIER.scope(std::sync::Arc::clone(&barrier), store.load_all(&cache));
let concurrent = async {
barrier.0.notified().await;
cache.add_or_update_sts_account("new-session", &stale, time::OffsetDateTime::now_utc());
cache.add_or_update_user("new-service", &stale, time::OffsetDateTime::now_utc());
barrier.1.notify_one();
};
let (loaded, ()) = tokio::join!(loading, concurrent);
loaded.expect("reconcile revoked identities despite STS traffic");
let snapshot = cache.snapshot();
assert!(!snapshot.users.contains_key("revoked-user"));
assert!(snapshot.users.contains_key("new-service"));
assert!(snapshot.sts_accounts.contains_key("new-session"));
assert!(snapshot.groups.is_empty());
assert!(snapshot.user_policies.is_empty());
assert!(snapshot.group_policies.is_empty());
assert!(snapshot.user_group_memberships.is_empty());
assert!(!snapshot.policy_docs.is_empty(), "built-in policies must remain available");
});
})
.expect("spawn IAM reload test thread")
.join()
.expect("IAM reload test should complete");
}
/// rustfs#4304 / startup contract rustfs#4056: the bulk snapshot load
/// (`load_all`) must never depend on the node-counted namespace lock
/// quorum, so its reads have to carry `no_lock = true` down to the
+18 -1013
View File
File diff suppressed because it is too large Load Diff
@@ -1411,28 +1411,6 @@ pub fn record_peer_reachable(addr: &str) {
publish_offline_gauge(&peers);
}
/// Record a successful interaction with a cluster peer, but only when the peer is currently
/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
/// and lock-path successes that do not go through a fresh dial.
///
/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
pub fn record_peer_reachable_if_offline(addr: &str) {
let key = normalize_peer_key(addr);
// Fast path: read lock to check whether the peer is currently offline.
{
let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
match peers.get(key) {
None => return,
Some(entry) if entry.online => return,
Some(_) => {} // offline: fall through to the write path
}
}
// Slow path: the peer is offline, so flip it back online and republish the gauge.
record_peer_reachable(addr);
}
/// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
/// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
@@ -2210,34 +2188,6 @@ mod tests {
assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
}
#[test]
fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
let addr = "http://cluster-peer-recover-if-offline-test:9000";
// Mark the peer offline.
record_peer_unreachable(addr, 1);
assert!(cluster_peer_is_offline(addr));
// A successful cached-channel RPC (no fresh dial) clears the offline state.
record_peer_reachable_if_offline(addr);
assert!(!cluster_peer_is_offline(addr));
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
}
#[test]
fn record_peer_reachable_if_offline_no_op_for_online_peer() {
let addr = "http://cluster-peer-recover-noop-test:9000";
record_peer_reachable(addr);
// Should not panic or change state for an already-online peer.
record_peer_reachable_if_offline(addr);
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
// Should also be a no-op for unknown peers.
record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
}
#[test]
fn cluster_peer_should_bypass_is_self_healing() {
let addr = "http://cluster-peer-bypass-selfheal-test:9000";
-11
View File
@@ -85,17 +85,6 @@ impl KeystoneAuthProvider {
Ok(self.keystone_token_to_credentials(&keystone_token))
}
/// Revalidate a long-lived subscription directly with Keystone, bypassing
/// the request cache so token revocation terminates an existing stream.
/// Returns the same authentication and expiry errors as a new token login.
pub async fn revalidate_subscription_token(&self, token: &str) -> Result<Credentials> {
let validated = self.client.validate_token(token).await?;
if validated.is_expired() {
return Err(KeystoneError::TokenExpired);
}
Ok(self.keystone_token_to_credentials(&validated))
}
/// Authenticate using EC2 credentials (S3 API with AWS SigV4)
pub async fn authenticate_with_ec2(&self, access_key: &str, signature: &str, string_to_sign: &str) -> Result<Credentials> {
// Check cache
+6 -142
View File
@@ -121,7 +121,7 @@ impl KeystoneClient {
.await
.map_err(|e| {
error!("Failed to send token validation request: {}", e);
KeystoneError::HttpError(e.without_url().to_string())
KeystoneError::HttpError(e.to_string())
})?;
let status = response.status();
@@ -138,10 +138,7 @@ impl KeystoneClient {
)));
}
let body: serde_json::Value = response
.json()
.await
.map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
self.parse_token_v3(&body)
}
@@ -263,16 +260,13 @@ impl KeystoneClient {
.header("X-Auth-Token", admin_token)
.send()
.await
.map_err(|e| KeystoneError::HttpError(e.without_url().to_string()))?;
.map_err(|e| KeystoneError::HttpError(e.to_string()))?;
if !response.status().is_success() {
return Ok(vec![]);
}
let body: serde_json::Value = response
.json()
.await
.map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
let credentials = body
.get("credentials")
@@ -371,7 +365,7 @@ impl KeystoneClient {
.json(&auth_payload)
.send()
.await
.map_err(|e| KeystoneError::HttpError(e.without_url().to_string()))?;
.map_err(|e| KeystoneError::HttpError(e.to_string()))?;
if !response.status().is_success() {
return Err(KeystoneError::AuthenticationFailed("Admin authentication failed".to_string()));
@@ -385,10 +379,7 @@ impl KeystoneClient {
.to_string();
// Parse expiration from response body
let body: serde_json::Value = response
.json()
.await
.map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
let expires_at = body
.get("token")
@@ -419,133 +410,6 @@ impl KeystoneClient {
mod tests {
use super::*;
#[tokio::test]
async fn subscription_token_refresh_bypasses_a_still_valid_login_cache() {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("fake Keystone");
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let expires = (OffsetDateTime::now_utc() + time::Duration::hours(1))
.format(&time::format_description::well_known::Rfc3339)
.unwrap();
let body = json!({"token":{"user":{"id":"listener","name":"listener"},
"roles":[{"name":"Member"}],"expires_at":expires}})
.to_string();
let server = std::thread::spawn(move || {
for response in [
format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
),
"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
] {
let (mut socket, _) = listener.accept().expect("token validation request");
socket.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap();
let mut request = [0; 4096];
let length = socket.read(&mut request).expect("request headers");
assert!(String::from_utf8_lossy(&request[..length]).contains("subscription-token"));
socket.write_all(response.as_bytes()).expect("validation response");
}
});
let mut client = KeystoneClient::new(
endpoint,
KeystoneVersion::V3,
None,
None,
None,
"Default".into(),
true,
std::time::Duration::from_secs(5),
);
client.client = reqwest::Client::builder()
.no_proxy()
.timeout(std::time::Duration::from_secs(5))
.build()
.unwrap();
let provider = crate::KeystoneAuthProvider::new(client, 10, std::time::Duration::from_secs(60), true);
assert!(provider.authenticate_with_token("subscription-token").await.is_ok());
assert!(
provider.authenticate_with_token("subscription-token").await.is_ok(),
"ordinary login still uses the valid cached token"
);
assert!(matches!(
provider.revalidate_subscription_token("subscription-token").await,
Err(KeystoneError::InvalidToken)
));
server.join().expect("fake Keystone finished");
}
#[tokio::test]
async fn middleware_keystone_failures_do_not_disclose_upstream_locations() {
use http_body_util::{BodyExt, Full};
use std::io::{Read, Write};
use tower::{Layer, ServiceExt};
for (response, details) in [
(None, "Keystone token validation failed"),
(
Some("HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nnot-json"),
"Keystone token validation failed",
),
(
Some("HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"),
"Invalid token",
),
] {
for header in ["X-Auth-Token", "X-Storage-Token"] {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind fake Keystone");
let endpoint = format!("http://{}/private-deployment", listener.local_addr().unwrap());
let server = response.map(|response| {
std::thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("Keystone request");
stream.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap();
let mut request = [0; 4096];
assert!(stream.read(&mut request).expect("read token validation request") > 0);
stream.write_all(response.as_bytes()).expect("send fake response");
})
});
let mut client = KeystoneClient::new(
endpoint.clone(),
KeystoneVersion::V3,
None,
None,
None,
"Default".to_owned(),
true,
std::time::Duration::from_secs(5),
);
client.client = reqwest::Client::builder()
.no_proxy()
.timeout(std::time::Duration::from_secs(5))
.build()
.unwrap();
let provider = crate::KeystoneAuthProvider::new(client, 1, std::time::Duration::from_secs(1), false);
let inner = tower::service_fn(|_| async {
Ok::<_, std::convert::Infallible>(http::Response::new(Full::new(bytes::Bytes::new())))
});
let response = crate::middleware::KeystoneAuthLayer::new(Some(Arc::new(provider)))
.layer(inner)
.oneshot(
http::Request::builder()
.header(header, "private-token")
.body(Full::new(bytes::Bytes::new()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), http::StatusCode::UNAUTHORIZED);
let body = response.into_body().collect().await.unwrap().to_bytes();
let body = std::str::from_utf8(&body).unwrap();
assert!(body.contains(&format!("<Details>{details}</Details>")));
for secret in [&endpoint, "private-deployment", "private-token", "/v3/auth/tokens"] {
assert!(!body.contains(secret), "upstream diagnostics must not appear in the response");
}
if let Some(server) = server {
server.join().unwrap();
}
}
}
}
#[test]
fn test_client_creation() {
let client = KeystoneClient::new(
+1 -5
View File
@@ -173,11 +173,7 @@ where
<Message>Invalid Keystone token</Message>
<Details>{}</Details>
</Error>"#,
xml_escape(match e {
crate::KeystoneError::InvalidToken => "Invalid token",
crate::KeystoneError::TokenExpired => "Token expired",
_ => "Keystone token validation failed",
})
xml_escape(&e.to_string())
);
let body: BoxBody = Full::new(Bytes::from(error_xml))
+1 -11
View File
@@ -214,9 +214,6 @@ impl KmsManager {
}
async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
if request.key_name.as_deref() == Some(crate::probe::PROBE_KEY_ID) {
return Err(KmsError::invalid_operation("the KMS probe key is reserved for internal health checks"));
}
// A blank name is neither "generate one" (that is `None`) nor a usable
// id: the Local backend would write a key file with an empty stem and
// the Vault backends would address their mount root, each failing with
@@ -441,9 +438,6 @@ impl KmsManager {
/// any backend sees the request. The backends repeat the window bound as a
/// defensive assertion for callers that hold a backend handle directly.
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
if request.key_id == crate::probe::PROBE_KEY_ID {
return Err(KmsError::invalid_operation("the KMS probe key cannot be deleted through key management"));
}
self.check_deletion_request(&request)?;
if request.force_immediate.unwrap_or(false) {
self.refuse_referenced_immediate_deletion(&request.key_id).await?;
@@ -597,11 +591,7 @@ impl KmsManager {
/// Disable a key on behalf of `context`'s principal
pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
let started = Instant::now();
let result = if key_id == crate::probe::PROBE_KEY_ID {
Err(KmsError::invalid_operation("the KMS probe key cannot be disabled through key management"))
} else {
self.backend.disable_key(key_id).await
};
let result = self.backend.disable_key(key_id).await;
if result.is_ok() {
self.invalidate_cached_metadata(key_id).await;
}
-51
View File
@@ -700,57 +700,6 @@ mod tests {
);
}
#[tokio::test]
async fn key_management_cannot_preempt_or_disable_the_readiness_probe() {
let temp_dir = tempfile::tempdir().expect("key directory");
let backend = local_backend(&temp_dir).await;
let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
config.allow_immediate_deletion = true;
let manager = crate::manager::KmsManager::new(backend.clone(), config);
let request = CreateKeyRequest {
key_name: Some(PROBE_KEY_ID.to_owned()),
..Default::default()
};
assert!(matches!(manager.create_key(request).await, Err(KmsError::InvalidOperation { .. })));
let worker = worker(backend.clone());
assert_eq!(worker.round().await, ProbeResult::Success, "the probe still provisions its reserved key");
assert!(matches!(manager.disable_key(PROBE_KEY_ID).await, Err(KmsError::InvalidOperation { .. })));
for immediate in [false, true] {
assert!(matches!(
manager
.delete_key(DeleteKeyRequest {
key_id: PROBE_KEY_ID.to_owned(),
pending_window_in_days: Some(7),
force_immediate: Some(immediate),
confirm_key_id: Some(PROBE_KEY_ID.to_owned())
})
.await,
Err(KmsError::InvalidOperation { .. })
));
assert_eq!(worker.round().await, ProbeResult::Success);
}
let customer = manager
.create_key(CreateKeyRequest {
key_name: Some("customer-probe-control".to_owned()),
..Default::default()
})
.await
.expect("customer key");
manager
.disable_key(&customer.key_id)
.await
.expect("customer key can be disabled");
manager
.delete_key(DeleteKeyRequest {
key_id: customer.key_id,
pending_window_in_days: Some(7),
..Default::default()
})
.await
.expect("customer deletion remains available");
assert_eq!(worker.round().await, ProbeResult::Success);
}
#[test]
fn read_only_backend_is_reported_unsupported_without_failures() {
let (snapshot, result) = record_metrics(|| {
-51
View File
@@ -41,16 +41,6 @@ pub(super) fn pyroscope_log_filter() -> FilterFn {
filter_fn(allows_non_pyroscope_dependency_event)
}
/// s3s verbose events include response bodies and expected signatures. Keep
/// diagnostics spans, but never let these events reach logs or OTLP exporters.
pub(super) fn s3s_log_filter() -> FilterFn {
filter_fn(|metadata| {
!(metadata.is_event()
&& (metadata.target() == "s3s" || metadata.target().starts_with("s3s::"))
&& *metadata.level() > tracing::Level::INFO)
})
}
/// Build an `EnvFilter` from the given log level string.
///
/// If `default_level` is provided, it is used directly. Otherwise, the
@@ -349,47 +339,6 @@ mod tests {
format!("{HTTP_SERVER_LOG_TARGET}={level}")
}
#[test]
fn s3s_filter_blocks_credentials_despite_explicit_verbose_directives() {
use tracing_subscriber::prelude::*;
for directive in ["trace", concat!("trace,", "s3s::service=trace,", "s3s::ops::signature=debug")] {
let writer = SharedWriter::default();
let captured = Arc::clone(&writer.0);
let subscriber = tracing_subscriber::registry()
.with(EnvFilter::new(directive))
.with(s3s_log_filter())
.with(
tracing_subscriber::fmt::layer()
.without_time()
.with_ansi(false)
.with_writer(writer)
.with_span_events(tracing_subscriber::fmt::format::FmtSpan::NEW),
);
with_default(subscriber, || {
let span = tracing::debug_span!(target: "s3s::service", "safe-operation-span");
let _entered = span.enter();
tracing::debug!(target: "s3s::service", response = "secret-response", "response dump");
tracing::trace!(target: "s3s", response = "secret-trace", "response dump");
tracing::debug!(target: "s3s::ops::signature", expected = "secret-signature", "signature mismatch");
tracing::info!(target: "s3s::service", "visible-info");
tracing::warn!(target: "s3s::service", "visible-warning");
tracing::debug!(target: "rustfs", "visible-rustfs-debug");
tracing::debug!(target: "s3s_other", "visible-other-target");
});
let output = String::from_utf8(captured.lock().expect("captured logs").clone()).expect("UTF-8 logs");
assert!(!output.contains("secret"), "{output}");
for expected in [
"safe-operation-span",
"visible-info",
"visible-warning",
"visible-rustfs-debug",
"visible-other-target",
] {
assert!(output.contains(expected), "missing {expected}: {output}");
}
}
}
#[test]
fn test_is_verbose_level() {
assert!(is_verbose_level("debug"));
+1 -3
View File
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
use crate::config::OtelConfig;
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter};
use crate::telemetry::rolling::{RollingAppender, Rotation};
use metrics::counter;
use rustfs_config::observability::{
@@ -369,7 +369,6 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(ErrorLayer::default())
.with(fmt_layer)
.try_init()
@@ -487,7 +486,6 @@ fn init_file_logging_internal(
tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(ErrorLayer::default())
.with(file_layer)
.with(stdout_layer)
+1 -4
View File
@@ -39,7 +39,7 @@
use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::set_observability_metric_enabled;
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -296,7 +296,6 @@ pub(super) fn init_observability_http(
tracing_subscriber::registry()
.with(filter)
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(ErrorLayer::default())
.with(file_layer_opt)
.with(stdout_layer_opt)
@@ -1197,7 +1196,6 @@ mod tests {
let subscriber = tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(
tracing_subscriber::fmt::layer()
.with_ansi(false)
@@ -1221,7 +1219,6 @@ mod tests {
tracing_subscriber::registry()
.with(build_env_filter("info", None))
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
.try_init()
.expect("install isolated LogTracer subscriber");
+1 -1
View File
@@ -20,7 +20,7 @@ const ARN_PARTITION_RUSTFS: &str = "rustfs";
const ARN_SERVICE_IAM: &str = "iam";
const ARN_RESOURCE_TYPE_ROLE: &str = "role";
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct ARN {
pub partition: String,
pub service: String,
+15 -32
View File
@@ -165,10 +165,9 @@ impl<'de> Deserialize<'de> for ActionSet {
}
}
#[derive(Deserialize, Hash, PartialEq, Eq, Clone, Debug, Copy)]
#[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, Debug, Copy)]
#[serde(try_from = "&str", untagged)]
pub enum Action {
All,
S3Action(S3Action),
AdminAction(AdminAction),
StsAction(StsAction),
@@ -176,29 +175,12 @@ pub enum Action {
None,
}
impl Serialize for Action {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
Self::None => serializer.serialize_unit(),
_ => serializer.serialize_str(self.into()),
}
}
}
impl Action {
pub fn is_match(&self, action: &Action) -> bool {
self.is_match_for_effect(action, false)
}
pub fn is_match_for_effect(&self, action: &Action, deny: bool) -> bool {
if matches!(self, Self::All) {
// Preserve the historical S3-only Allow while allowing deny-all and
// NotAction exclusions to cover every action family.
return deny || Self::S3Action(S3Action::AllActions).is_match_for_effect(action, false);
}
// Force-delete bypasses emptiness and version checks. A blanket `s3:*`
// / `*` Allow (including canned consoleAdmin) must not confer it; the
// statement has to name `s3:ForceDeleteBucket` or `s3:ForceDeleteObject`.
@@ -220,7 +202,6 @@ fn action_requires_explicit_grant(action: &Action) -> bool {
impl From<&Action> for &str {
fn from(value: &Action) -> &'static str {
match value {
Action::All => "*",
Action::S3Action(s) => s.into(),
Action::AdminAction(s) => s.into(),
Action::StsAction(s) => s.into(),
@@ -240,8 +221,9 @@ impl Action {
impl TryFrom<&str> for Action {
type Error = Error;
fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
// Support wildcard "*" which matches all S3 actions (AWS S3 standard)
if value == "*" {
return Ok(Self::All);
return Ok(Self::S3Action(S3Action::AllActions));
}
if value.starts_with(Self::S3_PREFIX) {
Ok(Self::S3Action(
@@ -840,7 +822,7 @@ mod tests {
#[test]
fn blanket_s3_wildcard_does_not_grant_force_delete() {
let wildcard = Action::try_from("s3:*").expect("s3:* parses");
let star = Action::try_from("*").expect("* parses");
let star = Action::try_from("*").expect("* parses as s3:*");
let force_bucket = Action::try_from("s3:ForceDeleteBucket").expect("force bucket action parses");
let force_object = Action::try_from("s3:ForceDeleteObject").expect("force object action parses");
let delete_object = Action::try_from("s3:DeleteObject").expect("delete object parses");
@@ -859,10 +841,9 @@ mod tests {
#[test]
fn test_action_wildcard_parsing() {
// Preserve the bare wildcard through serialization.
// Test that "*" parses to S3Action::AllActions
let action = Action::try_from("*").expect("Should parse wildcard");
assert!(matches!(action, Action::All));
assert_eq!(serde_json::to_string(&action).expect("serialize wildcard"), r#""*""#);
assert!(matches!(action, Action::S3Action(S3Action::AllActions)));
}
#[test]
@@ -948,13 +929,15 @@ mod tests {
#[test]
fn test_actionset_wildcard_serialization() {
for wildcard in ["*", "s3:*"] {
let actionset = ActionSet(vec![Action::try_from(wildcard).expect("parse wildcard")]);
let json = serde_json::to_string(&actionset).expect("serialize action set");
assert_eq!(json, format!(r#"["{wildcard}"]"#));
let round_trip: ActionSet = serde_json::from_str(&json).expect("deserialize action set");
assert_eq!(round_trip, actionset);
}
// Wildcard action should serialize as array for S3 specification compliance
let actionset = ActionSet(vec![Action::try_from("*").expect("Should parse wildcard")]);
let json = serde_json::to_string(&actionset).expect("Should serialize");
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
assert!(parsed.is_array(), "Should serialize as array");
let arr = parsed.as_array().expect("Should be array");
assert_eq!(arr.len(), 1);
assert_eq!(arr[0].as_str().unwrap(), "s3:*");
}
#[test]
+15 -52
View File
@@ -15,7 +15,7 @@
use super::func::InnerFunc;
use ipnetwork::IpNetwork;
use serde::{Deserialize, Serialize, de::Visitor};
use std::{collections::HashMap, net::IpAddr};
use std::{borrow::Cow, collections::HashMap, net::IpAddr};
pub type AddrFunc = InnerFunc<AddrFuncValue>;
@@ -30,16 +30,7 @@ impl AddrFunc {
};
for ip_net in inner.values.0.iter() {
let canonical = ip.to_canonical();
let matches_mapped_network = match (ip_net, canonical) {
(IpNetwork::V6(network), IpAddr::V4(address))
if network.prefix() >= 96 && network.ip().to_ipv4_mapped().is_some() =>
{
network.contains(address.to_ipv6_mapped())
}
_ => false,
};
if ip_net.contains(ip) || ip_net.contains(canonical) || matches_mapped_network {
if ip_net.contains(ip) {
return true;
}
}
@@ -90,7 +81,13 @@ impl<'de> Deserialize<'de> for AddrFuncValue {
impl AddrFuncValueVisitor {
fn cidr<E: serde::de::Error>(v: &str) -> Result<IpNetwork, E> {
v.parse::<IpNetwork>()
let mut cidr_str = Cow::from(v);
if v.find('/').is_none() {
cidr_str.to_mut().push_str("/32");
}
cidr_str
.parse::<IpNetwork>()
.map_err(|_| E::custom(format!("{v} can not be parsed to CIDR")))
}
}
@@ -110,40 +107,6 @@ mod tests {
};
use test_case::test_case;
#[test]
fn source_ip_matches_ipv4_and_mapped_forms_across_protocols() {
for network in ["203.0.113.0/24", "::ffff:203.0.113.0/120"] {
let function: AddrFunc =
serde_json::from_str(&format!(r#"{{"aws:SourceIp":"{network}"}}"#)).expect("parse source IP condition");
for address in ["203.0.113.9", "::ffff:203.0.113.9"] {
assert!(
function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![address.to_string()]),])),
"{network} must match {address}"
);
}
for address in ["198.51.100.9", "::ffff:198.51.100.9", "2001:db8::9"] {
assert!(
!function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![address.to_string()]),])),
"{network} must not match {address}"
);
}
}
}
#[test]
fn bare_address_matches_only_the_named_host() {
for (host, neighbor, prefix) in [
("2001:db8:85a3::8a2e:370:7334", "2001:db8:ffff::1", 128),
("203.0.113.7", "203.0.113.8", 32),
] {
let function: AddrFunc =
serde_json::from_str(&format!(r#"{{"aws:SourceIp":"{host}"}}"#)).expect("parse bare IP address condition");
assert_eq!(function.0[0].values.0[0].prefix(), prefix);
assert!(function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![host.to_string()])])));
assert!(!function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![neighbor.to_string()])])));
}
}
fn new_func(name: KeyName, variable: Option<String>, value: Vec<&str>) -> AddrFunc {
AddrFunc {
0: vec![FuncKeyValue {
@@ -156,20 +119,20 @@ mod tests {
#[test_case(r#"{"aws:SourceIp": "203.0.113.0/24"}"#, new_func(Aws(AWSSourceIP), None, vec!["203.0.113.0/24"]); "1")]
#[test_case(r#"{"aws:SourceIp": "203.0.113.0"}"#, new_func(Aws(AWSSourceIP), None, vec!["203.0.113.0/32"]); "2")]
#[test_case(r#"{"aws:SourceIp": "2001:DB8:1234:5678::/64"}"#, new_func(Aws(AWSSourceIP),None, vec!["2001:DB8:1234:5678::/64"]); "3")]
#[test_case(r#"{"aws:SourceIp": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/128"]); "4")]
#[test_case(r#"{"aws:SourceIp": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/32"]); "4")]
#[test_case(r#"{"aws:SourceIp": ["203.0.113.0/24","203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), None, vec!["203.0.113.0/24", "203.0.113.0/32"]); "5")]
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::/64","203.0.113.0/24"]}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/64", "203.0.113.0/24"]); "6")]
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),None, vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/128"]); "7")]
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/128", "203.0.113.0/32"]); "8")]
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),None, vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/32"]); "7")]
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/32", "203.0.113.0/32"]); "8")]
#[test_case(r#"{"aws:SourceIp/a": "203.0.113.0/24"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/24"]); "9")]
#[test_case(r#"{"aws:SourceIp/a": "203.0.113.0/24"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/24"]); "10")]
#[test_case(r#"{"aws:SourceIp/a": "203.0.113.0"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/32"]); "11")]
#[test_case(r#"{"aws:SourceIp/a": "2001:DB8:1234:5678::/64"}"#, new_func(Aws(AWSSourceIP),Some("a".into()), vec!["2001:DB8:1234:5678::/64"]); "12")]
#[test_case(r#"{"aws:SourceIp/a": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/128"]); "13")]
#[test_case(r#"{"aws:SourceIp/a": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/32"]); "13")]
#[test_case(r#"{"aws:SourceIp/a": ["203.0.113.0/24", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/24", "203.0.113.0/32"]); "14")]
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::/64", "203.0.113.0/24"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/64", "203.0.113.0/24"]); "15")]
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),Some("a".into()), vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/128"]); "16")]
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/128", "203.0.113.0/32"]); "17")]
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),Some("a".into()), vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/32"]); "16")]
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/32", "203.0.113.0/32"]); "17")]
fn test_deser(input: &str, expect: AddrFunc) -> Result<(), serde_json::Error> {
let v: AddrFunc = serde_json::from_str(input)?;
assert_eq!(v, expect);
+7 -37
View File
@@ -61,11 +61,6 @@ pub enum Condition {
impl Condition {
pub fn from_deserializer<'a, D: MapAccess<'a>>(key: &str, d: &mut D) -> Result<Self, D::Error> {
// Two suffixes are retained for existing serialized policies. Bound the
// chain before parsing so Clone, evaluation and Drop stay bounded too.
if key.ends_with("IfExistsIfExistsIfExists") {
return Err(Error::custom("condition operator has too many IfExists suffixes"));
}
Ok(match key {
"StringEquals" => Self::StringEquals(d.next_value()?),
"StringNotEquals" => Self::StringNotEquals(d.next_value()?),
@@ -267,18 +262,18 @@ impl Condition {
Null(s) => s.evaluate_null(values),
Bool(s) => s.evaluate_bool(values),
NumericEquals(s) => s.evaluate(i64::eq, false, values),
NumericNotEquals(s) => s.evaluate(i64::ne, true, values),
NumericNotEquals(s) => s.evaluate(i64::ne, false, values),
NumericLessThan(s) => s.evaluate(i64::lt, false, values),
NumericLessThanEquals(s) => s.evaluate(i64::le, false, values),
NumericGreaterThan(s) => s.evaluate(i64::gt, false, values),
NumericGreaterThanIfExists(s) => s.evaluate(i64::ge, true, values),
NumericGreaterThanEquals(s) => s.evaluate(i64::ge, false, values),
DateEquals(s) => s.evaluate(OffsetDateTime::eq, false, values),
DateNotEquals(s) => s.evaluate(OffsetDateTime::ne, true, values),
DateLessThan(s) => s.evaluate(OffsetDateTime::lt, false, values),
DateLessThanEquals(s) => s.evaluate(OffsetDateTime::le, false, values),
DateGreaterThan(s) => s.evaluate(OffsetDateTime::gt, false, values),
DateGreaterThanEquals(s) => s.evaluate(OffsetDateTime::ge, false, values),
DateEquals(s) => s.evaluate(OffsetDateTime::eq, values),
DateNotEquals(s) => s.evaluate(OffsetDateTime::ne, values),
DateLessThan(s) => s.evaluate(OffsetDateTime::lt, values),
DateLessThanEquals(s) => s.evaluate(OffsetDateTime::le, values),
DateGreaterThan(s) => s.evaluate(OffsetDateTime::gt, values),
DateGreaterThanEquals(s) => s.evaluate(OffsetDateTime::ge, values),
IfExists(inner) => {
if !inner.has_any_key_in(values) {
return true;
@@ -491,31 +486,6 @@ mod tests {
assert_eq!(functions, deserialized);
}
#[test]
fn repeated_if_exists_suffixes_are_rejected_before_recursion() {
for count in [3, 10_000] {
let key = format!("StringEquals{}", "IfExists".repeat(count));
let document = serde_json::json!({key: {"s3:x-amz-server-side-encryption": "aws:kms"}});
let result = serde_json::from_slice::<Functions>(&serde_json::to_vec(&document).expect("serialize condition"));
assert!(result.is_err(), "{count} suffixes must be rejected without constructing a deep chain");
}
}
#[tokio::test]
async fn numeric_and_date_not_equals_match_absent_keys() {
for (operator, key, value, absent) in [
("NumericNotEquals", "s3:max-keys", "30", true),
("NumericEquals", "s3:max-keys", "30", false),
("DateNotEquals", "s3:object-lock-retain-until-date", "2030-01-01T00:00:00Z", true),
("DateEquals", "s3:object-lock-retain-until-date", "2030-01-01T00:00:00Z", false),
] {
let document = serde_json::json!({operator: {key: value}});
let functions: Functions =
serde_json::from_slice(&serde_json::to_vec(&document).expect("serialize condition")).expect("parse condition");
assert_eq!(functions.evaluate(&HashMap::new()).await, absent, "{operator}");
}
}
#[test]
fn test_nested_if_exists_serde_round_trip() {
// Verifies that nested IfExists(IfExists(StringEquals)) serializes with the
+7 -56
View File
@@ -13,7 +13,6 @@
// limitations under the License.
use super::func::InnerFunc;
use super::key_name::{AwsKeyName, KeyName};
use serde::{Deserialize, Deserializer, Serialize, de};
use std::{collections::HashMap, fmt};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
@@ -21,27 +20,14 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
pub type DateFunc = InnerFunc<DateFuncValue>;
impl DateFunc {
pub fn evaluate(
&self,
op: impl Fn(&OffsetDateTime, &OffsetDateTime) -> bool,
absent: bool,
values: &HashMap<String, Vec<String>>,
) -> bool {
pub fn evaluate(&self, op: impl Fn(&OffsetDateTime, &OffsetDateTime) -> bool, values: &HashMap<String, Vec<String>>) -> bool {
for inner in self.0.iter() {
let v = match values.get(inner.key.name().as_str()).and_then(|x| x.first()) {
Some(x) => x,
None if absent => continue,
None => return false,
};
let parsed = if inner.key.is(&KeyName::Aws(AwsKeyName::AWSEpochTime)) {
v.parse::<i64>()
.ok()
.and_then(|seconds| OffsetDateTime::from_unix_timestamp(seconds).ok())
} else {
OffsetDateTime::parse(v, &Rfc3339).ok()
};
let Some(rv) = parsed else {
let Ok(rv) = OffsetDateTime::parse(v, &Rfc3339) else {
return false;
};
@@ -102,7 +88,7 @@ impl<'de> Deserialize<'de> for DateFuncValue {
#[cfg(test)]
mod tests {
use super::{AwsKeyName, DateFunc, DateFuncValue};
use super::{DateFunc, DateFuncValue};
use crate::policy::function::func::FuncKeyValue;
use crate::policy::function::{
key::Key,
@@ -138,50 +124,15 @@ mod tests {
Ok(())
}
#[test]
fn date_conditions_compare_epoch_seconds_only_for_epoch_time() {
let function = new_func(KeyName::Aws(AwsKeyName::AWSEpochTime), None, "2020-01-01T00:00:00Z");
for (value, equal, later) in [
("1577836799", false, false),
("1577836800", true, false),
("1577836801", false, true),
] {
let values = HashMap::from([("EpochTime".to_string(), vec![value.to_string()])]);
assert_eq!(function.evaluate(OffsetDateTime::eq, false, &values), equal);
assert_eq!(function.evaluate(OffsetDateTime::gt, false, &values), later);
assert_eq!(function.evaluate(OffsetDateTime::lt, false, &values), !equal && !later);
}
for invalid in ["2020-01-01T00:00:00Z", "invalid", "9223372036854775807"] {
let values = HashMap::from([("EpochTime".to_string(), vec![invalid.to_string()])]);
assert!(!function.evaluate(OffsetDateTime::ne, true, &values));
}
let header_date = new_func(S3(S3ObjectLockRetainUntilDate), None, "2020-01-01T00:00:00Z");
let values = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["1577836800".to_string()])]);
assert!(!header_date.evaluate(OffsetDateTime::eq, false, &values));
}
#[test]
fn absent_date_key_does_not_skip_other_keys() {
let mut function = new_func(S3(S3ObjectLockRetainUntilDate), None, "2030-01-01T00:00:00Z");
function
.0
.extend(new_func(KeyName::Aws(AwsKeyName::AWSCurrentTime), None, "2030-01-01T00:00:00Z").0);
let values = HashMap::from([("CurrentTime".to_string(), vec!["2030-01-01T00:00:00Z".to_string()])]);
assert!(!function.evaluate(OffsetDateTime::ne, true, &values));
assert!(function.evaluate(OffsetDateTime::eq, true, &values));
function.0.reverse();
assert!(!function.evaluate(OffsetDateTime::ne, true, &values));
}
#[test]
fn evaluate_compares_request_date_to_policy_date() {
let function = new_func(S3(S3ObjectLockRetainUntilDate), None, "2030-01-01T00:00:00Z");
let later = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2099-01-01T00:00:00Z".to_string()])]);
let earlier = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2029-01-01T00:00:00Z".to_string()])]);
assert!(function.evaluate(OffsetDateTime::gt, false, &later));
assert!(!function.evaluate(OffsetDateTime::gt, false, &earlier));
assert!(function.evaluate(OffsetDateTime::lt, false, &earlier));
assert!(!function.evaluate(OffsetDateTime::lt, false, &later));
assert!(function.evaluate(OffsetDateTime::gt, &later));
assert!(!function.evaluate(OffsetDateTime::gt, &earlier));
assert!(function.evaluate(OffsetDateTime::lt, &earlier));
assert!(!function.evaluate(OffsetDateTime::lt, &later));
}
}
+2 -17
View File
@@ -26,12 +26,11 @@ pub type NumberFunc = InnerFunc<NumberFuncValue>;
pub struct NumberFuncValue(i64);
impl NumberFunc {
pub fn evaluate(&self, op: impl Fn(&i64, &i64) -> bool, absent: bool, values: &HashMap<String, Vec<String>>) -> bool {
pub fn evaluate(&self, op: impl Fn(&i64, &i64) -> bool, if_exists: bool, values: &HashMap<String, Vec<String>>) -> bool {
for inner in self.0.iter() {
let v = match values.get(inner.key.name().as_str()).and_then(|x| x.first()) {
Some(x) => x,
None if absent => continue,
None => return false,
None => return if_exists,
};
let Ok(rv) = v.parse::<i64>() else {
@@ -147,20 +146,6 @@ mod tests {
assert_eq!(limit.evaluate(i64::eq, false, &values), equals);
}
#[test]
fn absent_numeric_key_does_not_skip_other_keys() {
let mut function = new_func(S3(S3MaxKeys), None, 30);
function
.0
.extend(new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30).0);
let values = HashMap::from([("object-lock-remaining-retention-days".to_string(), vec!["30".to_string()])]);
assert!(!function.evaluate(i64::ne, true, &values));
assert!(!function.evaluate(i64::gt, true, &values));
assert!(function.evaluate(i64::eq, true, &values));
function.0.reverse();
assert!(!function.evaluate(i64::ne, true, &values));
}
#[test]
fn remaining_retention_days_numeric_condition_without_value_uses_if_exists() {
let limit = new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30);
-271
View File
@@ -3416,277 +3416,6 @@ mod test {
assert_eq!(round_trip.statements[0].effect, policy.statements[0].effect);
}
#[tokio::test]
async fn global_wildcard_deny_covers_all_families_without_broadening_allow() {
for action in [
"s3:GetObject",
"s3:ForceDeleteObject",
"admin:CreateUser",
"sts:AssumeRole",
"kms:Configure",
"kms:Decrypt",
] {
let parsed_action = Action::try_from(action).expect("action parses");
let conditions = HashMap::new();
let claims = HashMap::new();
let args = Args {
account: "user",
groups: &None,
action: parsed_action,
bucket: if action.starts_with("s3:") { "bucket" } else { "" },
object: if action.starts_with("s3:") { "object" } else { "" },
conditions: &conditions,
claims: &claims,
is_owner: false,
deny_only: false,
};
for (effect, selector, wildcard, expected) in [
("Deny", "Action", "*", false),
("Deny", "Action", "s3:*", !action.starts_with("s3:")),
("Deny", "NotAction", "*", true),
] {
let document = serde_json::json!({
"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": action, "Resource": "*"},
{"Effect": effect, selector: wildcard, "Resource": "*"}
]
});
let policy = Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy"))
.expect("parse wildcard policy");
assert_eq!(policy.is_allowed(&args).await, expected, "{effect} {selector} {wildcard} for {action}");
let serialized = serde_json::to_value(&policy).expect("serialize policy");
assert_eq!(serialized["Statement"][1][selector][0], wildcard);
}
for selector in ["Action", "NotAction"] {
let document = serde_json::json!({
"Version": "2012-10-17", "Statement": [{"Effect": "Allow", selector: "*", "Resource": "*"}]
});
let policy = Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy"))
.expect("parse wildcard Allow");
assert_eq!(policy.is_allowed(&args).await, selector == "Action" && action == "s3:GetObject");
}
}
}
#[tokio::test]
async fn global_wildcard_deny_still_obeys_resource_and_condition_scope() {
let document = serde_json::json!({
"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "admin:CreateUser", "Resource": "*"},
{"Effect": "Deny", "Action": "*", "Resource": "arn:aws:s3:::bucket/*"}
]
});
let policy = Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy")).expect("parse scoped Deny");
let claims = HashMap::new();
let conditions = HashMap::new();
let mut args = Args {
account: "user",
groups: &None,
action: Action::try_from("admin:CreateUser").expect("admin action"),
bucket: "",
object: "",
conditions: &conditions,
claims: &claims,
is_owner: false,
deny_only: false,
};
assert!(policy.is_allowed(&args).await, "bucket-scoped Deny must not become a global admin Deny");
let mut document = document;
document["Statement"][1]["Resource"] = serde_json::json!("*");
document["Statement"][1]["Condition"] = serde_json::json!({"StringEquals": {"aws:username": "blocked"}});
let conditional =
Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy")).expect("parse conditional Deny");
assert!(conditional.is_allowed(&args).await);
let conditions = HashMap::from([("username".to_string(), vec!["blocked".to_string()])]);
args.conditions = &conditions;
assert!(!conditional.is_allowed(&args).await);
}
#[tokio::test]
async fn security_conditions_enforce_identity_and_bucket_denies() {
for (operator, key, policy_value, request_value, denied) in [
("NumericNotEquals", "s3:object-lock-remaining-retention-days", "30", None, true),
("NumericNotEquals", "s3:object-lock-remaining-retention-days", "30", Some("30"), false),
("NumericNotEquals", "s3:object-lock-remaining-retention-days", "30", Some("10"), true),
("DateNotEquals", "s3:object-lock-retain-until-date", "2030-01-01T00:00:00Z", None, true),
(
"DateNotEquals",
"s3:object-lock-retain-until-date",
"2030-01-01T00:00:00Z",
Some("2030-01-01T00:00:00Z"),
false,
),
("DateGreaterThan", "aws:EpochTime", "2020-01-01T00:00:00Z", Some("1577836801"), true),
("DateGreaterThan", "aws:EpochTime", "2020-01-01T00:00:00Z", Some("1577836799"), false),
(
"NotIpAddress",
"aws:SourceIp",
"2001:db8:85a3::8a2e:370:7334",
Some("2001:db8:ffff::1"),
true,
),
(
"NotIpAddress",
"aws:SourceIp",
"2001:db8:85a3::8a2e:370:7334",
Some("2001:db8:85a3::8a2e:370:7334"),
false,
),
(
"IpAddress",
"aws:SourceIp",
"2001:db8:85a3::8a2e:370:7334",
Some("2001:db8:ffff::1"),
false,
),
("IpAddress", "aws:SourceIp", "2001:db8::/32", Some("2001:db8:ffff::1"), true),
] {
let mut document = serde_json::json!({
"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*"},
{"Effect": "Deny", "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*",
"Condition": {operator: {key: policy_value}}}
]
});
let policy =
Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy")).expect("parse identity policy");
let conditions = request_value
.map(|value| {
HashMap::from([(key.split_once(':').expect("condition prefix").1.to_string(), vec![value.to_string()])])
})
.unwrap_or_default();
let claims = HashMap::new();
let args = Args {
account: "user",
groups: &None,
action: Action::S3Action(S3Action::PutObjectAction),
bucket: "bucket",
object: "object",
conditions: &conditions,
claims: &claims,
is_owner: false,
deny_only: false,
};
assert_eq!(policy.is_allowed(&args).await, !denied, "identity {operator} {key} {request_value:?}");
for statement in document["Statement"].as_array_mut().expect("statements array") {
statement["Principal"] = serde_json::json!("*");
}
let policy: BucketPolicy = serde_json::from_slice(&serde_json::to_vec(&document).expect("serialize bucket policy"))
.expect("parse bucket policy");
assert_eq!(
policy
.is_allowed(&BucketPolicyArgs {
account: "user",
groups: &None,
action: args.action,
bucket: "bucket",
object: "object",
conditions: &conditions,
is_owner: false,
})
.await,
!denied,
"bucket {operator} {key} {request_value:?}"
);
}
}
#[tokio::test]
async fn service_principal_never_grants_access_to_a_named_access_key() {
for principal in [
serde_json::json!({"Service": "logging.s3.amazonaws.com"}),
serde_json::json!({"Service": "*"}),
serde_json::json!({"Service": "logging.s3.amazonaws.com", "AWS": "authorized-user"}),
] {
let document = serde_json::json!({
"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": principal,
"Action": ["s3:GetObject", "s3:PutObject"], "Resource": "arn:aws:s3:::bucket/*"}]
});
let policy: BucketPolicy = serde_json::from_slice(&serde_json::to_vec(&document).unwrap()).unwrap();
for action in [S3Action::GetObjectAction, S3Action::PutObjectAction] {
for account in ["logging.s3.amazonaws.com", "authorized-user"] {
let allowed = policy
.is_allowed(&BucketPolicyArgs {
account,
groups: &None,
action: Action::S3Action(action),
bucket: "bucket",
object: "log",
conditions: &HashMap::new(),
is_owner: false,
})
.await;
assert_eq!(allowed, account == "authorized-user" && principal.get("AWS").is_some());
}
}
assert_eq!(serde_json::to_value(&policy).unwrap()["Statement"][0]["Principal"], principal);
}
}
#[tokio::test]
async fn directory_objects_obey_prefix_denies_in_identity_and_bucket_policies() {
let mut document = serde_json::json!({
"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": "s3:*", "Resource": "arn:aws:s3:::bucket/*"},
{"Effect": "Deny", "Action": "s3:*", "Resource": "arn:aws:s3:::bucket/private/*"}
]
});
let identity = Policy::parse_config(&serde_json::to_vec(&document).unwrap()).unwrap();
for statement in document["Statement"].as_array_mut().unwrap() {
statement["Principal"] = serde_json::json!("*");
}
let bucket: BucketPolicy = serde_json::from_slice(&serde_json::to_vec(&document).unwrap()).unwrap();
for action in [
S3Action::GetObjectAction,
S3Action::PutObjectAction,
S3Action::DeleteObjectAction,
S3Action::DeleteObjectVersionAction,
] {
for (object, allowed) in [
("private/", false),
("private/child", false),
("public/", true),
("private", true),
] {
let conditions = HashMap::new();
assert_eq!(
identity
.is_allowed(&Args {
account: "user",
groups: &None,
action: Action::S3Action(action),
bucket: "bucket",
object,
conditions: &conditions,
claims: &HashMap::new(),
is_owner: false,
deny_only: false,
})
.await,
allowed,
"identity {action:?} {object}"
);
for account in ["user", ""] {
assert_eq!(
bucket
.is_allowed(&BucketPolicyArgs {
account,
groups: &None,
action: Action::S3Action(action),
bucket: "bucket",
object,
conditions: &conditions,
is_owner: false,
})
.await,
allowed,
"bucket {account} {action:?} {object}"
);
}
}
}
}
#[tokio::test]
async fn force_delete_actions_require_an_explicit_grant() {
let wildcard = Policy::parse_config(
+8 -3
View File
@@ -19,7 +19,7 @@ use std::collections::HashSet;
/// Principal that serializes AWS field as single string when containing only "*",
/// or as an array otherwise (matching AWS S3 API format).
/// Service principals round-trip for compatibility but do not match access keys.
/// Also supports Service principals (e.g., "logging.s3.amazonaws.com").
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Principal {
aws: HashSet<String>,
@@ -131,6 +131,11 @@ impl Principal {
return true;
}
}
for pattern in self.service.iter() {
if wildcard::is_simple_match(pattern, principal) {
return true;
}
}
false
}
}
@@ -220,7 +225,7 @@ mod test {
service: HashSet::from(["logging.s3.amazonaws.com".to_string()]),
};
assert!(!principal.is_match("logging.s3.amazonaws.com"));
assert!(principal.is_match("logging.s3.amazonaws.com"));
assert!(!principal.is_match("replication.s3.amazonaws.com"));
assert!(!principal.is_match("arn:aws:iam::123456789012:root"));
}
@@ -233,7 +238,7 @@ mod test {
};
assert!(principal.is_match("arn:aws:iam::123456789012:root"));
assert!(!principal.is_match("logging.s3.amazonaws.com"));
assert!(principal.is_match("logging.s3.amazonaws.com"));
assert!(!principal.is_match("other-principal"));
}
}
+13 -37
View File
@@ -17,11 +17,7 @@ use serde::{
Deserialize, Deserializer, Serialize,
de::{self, Error as DeError, Visitor},
};
use std::{
collections::{HashMap, HashSet},
fmt,
ops::Deref,
};
use std::{collections::HashMap, fmt, ops::Deref};
use super::{
Error as IamError, Validator,
@@ -54,6 +50,12 @@ impl ResourceSet {
self.0.is_empty()
}
fn push_unique(&mut self, resource: Resource) {
if !self.0.contains(&resource) {
self.0.push(resource);
}
}
pub async fn is_match(&self, resource: &str, conditions: &HashMap<String, Vec<String>>) -> bool {
self.is_match_with_resolver(resource, conditions, None).await
}
@@ -128,7 +130,9 @@ impl<'de> Deserialize<'de> for ResourceSet {
E: de::Error,
{
let resource = Resource::try_from(value).map_err(|e| E::custom(format!("invalid resource: {}", e)))?;
Ok(ResourceSet(vec![resource]))
let mut resources = ResourceSet::default();
resources.push_unique(resource);
Ok(resources)
}
fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
@@ -136,14 +140,11 @@ impl<'de> Deserialize<'de> for ResourceSet {
A: de::SeqAccess<'de>,
A::Error: DeError,
{
let mut resources = ResourceSet::default();
let mut seen = HashSet::new();
let mut resources = ResourceSet(Vec::with_capacity(seq.size_hint().unwrap_or(0)));
while let Some(value) = seq.next_element::<String>()? {
match Resource::try_from(value.as_str()) {
Ok(resource) => {
if seen.insert(resource.clone()) {
resources.0.push(resource);
}
resources.push_unique(resource);
}
Err(e) => {
return Err(A::Error::custom(format!("invalid resource: {}", e)));
@@ -223,14 +224,7 @@ impl Resource {
return true;
}
// Keep directory objects distinct from their bare key for prefix
// matching, while retaining path cleaning and exact-match compatibility.
let wildcard_resource = if matches!(self, Resource::S3(_)) && resource.ends_with('/') && cp.contains('/') {
format!("{cp}/")
} else {
cp.clone()
};
if cp != "." && wildcard::is_match(resolved_pattern, &wildcard_resource) {
if cp != "." && wildcard::is_match(resolved_pattern, &cp) {
return true;
}
}
@@ -320,20 +314,6 @@ mod tests {
use std::collections::HashMap;
use test_case::test_case;
#[test]
fn large_resource_sets_deduplicate_without_changing_first_seen_order() {
let unique = (0..40_000)
.map(|index| format!("arn:aws:s3:::bucket/key-{index}"))
.collect::<Vec<_>>();
let mut repeated = unique.clone();
repeated.extend(unique.iter().rev().cloned());
let resources: super::ResourceSet = serde_json::from_slice(&serde_json::to_vec(&repeated).expect("encode resources"))
.expect("large valid resource list parses");
assert_eq!(resources.len(), unique.len());
assert_eq!(serde_json::to_value(resources).expect("serialize resources"), serde_json::json!(unique));
assert!(serde_json::from_str::<super::ResourceSet>(r#"["arn:aws:s3:::bucket/key","invalid"]"#).is_err());
}
#[test_case("*", "mybucket" => true; "all buckets")]
#[test_case("*", "mybucket/myobject" => true; "all objects")]
#[test_case("*", "key/mykey" => true; "all keys")]
@@ -355,10 +335,6 @@ mod tests {
#[test_case("arn:aws:s3:::mybucket","mybucket/myobject" => false; "15")]
#[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/evil.txt" => false; "16")]
#[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/safe/../../victim-bucket/evil.txt" => false; "17")]
#[test_case("arn:aws:s3:::bucket/secret/*","bucket/secret/" => true; "directory prefix")]
#[test_case("arn:aws:s3:::bucket/secret","bucket/secret/" => true; "directory exact compatibility")]
#[test_case("arn:aws:s3:::bucket/*","bucket/" => false; "bucket is not a directory object")]
#[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/" => false; "directory traversal")]
#[test_case("arn:aws:kms:::key/mykey","key/mykey" => true; "kms exact key")]
#[test_case("arn:aws:kms:::key/mykey","key/otherkey" => false; "kms wrong key")]
#[test_case("arn:aws:kms:::key/*","key/mykey" => true; "kms key wildcard")]
+1 -1
View File
@@ -144,7 +144,7 @@ impl Statement {
for action in self.actions.iter() {
match action {
Action::All | Action::S3Action(_) => saw_s3 = true,
Action::S3Action(_) => saw_s3 = true,
Action::AdminAction(_) => saw_admin = true,
Action::StsAction(_) => saw_sts = true,
Action::KmsAction(_) => saw_kms = true,
+33 -21
View File
@@ -259,11 +259,35 @@ async fn resolve_aws_variables_with_depth(pattern: &str, resolver: &dyn PolicyVa
if depth >= 10 {
return vec![pattern.to_string()];
}
// Values from credentials and claims are data, not policy source. Expanding
// substituted placeholders again allows attacker-controlled exponential growth.
let mut results = resolve_single_pass(pattern, resolver, depth).await;
let mut seen = std::collections::HashSet::new();
results.retain(|result| seen.insert(result.clone()));
let mut results = vec![pattern.to_string()];
let mut changed = true;
let max_iterations = 10; // Prevent infinite loops
let mut iteration = 0;
while changed && iteration < max_iterations {
changed = false;
iteration += 1;
let mut new_results = Vec::new();
for result in &results {
let resolved = resolve_single_pass(result, resolver, depth).await;
if resolved.len() > 1 || (resolved.len() == 1 && &resolved[0] != result) {
changed = true;
}
new_results.extend(resolved);
}
// Remove duplicates while preserving order
results.clear();
let mut seen = std::collections::HashSet::new();
for result in new_results {
if seen.insert(result.clone()) {
results.push(result);
}
}
}
results
}
@@ -346,8 +370,8 @@ async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolve
let suffix = &results[i].0[end_pos + 1..];
for value in values {
// Resume after the substitution; generated placeholders
// are literal data and must never be expanded.
// Resume after the substitution; a generated placeholder
// belongs to the next bounded pass, not this one.
new_results.push((format!("{prefix}{value}{suffix}"), actual_pos + value.len()));
}
@@ -421,20 +445,8 @@ mod tests {
calls: AtomicUsize::new(0),
};
let result = resolve_aws_variables("${aws:AccountId}{aws:username}", &resolver).await;
assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
assert_eq!(result, vec!["$${aws:username}".to_string()]);
}
#[tokio::test]
async fn substituted_account_id_placeholders_are_literal() {
let account = "${aws:AccountId}".repeat(2);
let resolver = VariableResolver::new(VariableContext {
account_id: Some(account.clone()),
username: Some("alice".to_string()),
..Default::default()
});
let result = resolve_aws_variables("prefix/${aws:AccountId}/${aws:username}", &resolver).await;
assert_eq!(result, vec![format!("prefix/{account}/alice")]);
assert!(resolver.calls.load(Ordering::Relaxed) <= 10);
assert_eq!(result, vec!["${aws:AccountId}{aws:username}".to_string()]);
}
#[tokio::test]
-2
View File
@@ -30,7 +30,6 @@ workspace = true
[features]
default = []
fips = ["rustfs-tls-runtime?/fips"]
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
@@ -183,7 +182,6 @@ subtle = { workspace = true, optional = true }
socket2 = { workspace = true, optional = true, features = ["all"] }
[dev-dependencies]
rcgen = { workspace = true }
tempfile = { workspace = true }
proptest = "1"
rustfs-test-utils = { workspace = true }
-3
View File
@@ -16,9 +16,6 @@ pub mod client;
pub mod gateway;
pub mod session;
#[cfg(any(feature = "webdav", feature = "ftps"))]
pub(crate) mod tls;
#[cfg(test)]
pub(crate) mod dummy_storage;
-183
View File
@@ -1,183 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
use rustls::{ServerConfig, server::ResolvesServerCert};
use std::{io, path::Path, sync::Arc};
/// Shared listener TLS policy: a configured client CA always requires client authentication.
pub(crate) fn server_config(resolver: Arc<dyn ResolvesServerCert>, ca_file: Option<&str>) -> io::Result<ServerConfig> {
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let builder = ServerConfig::builder();
let builder = if let Some(ca_file) = ca_file {
builder.with_client_cert_verifier(rustfs_tls_runtime::build_webpki_client_verifier_from_ca_file(Path::new(ca_file))?)
} else {
builder.with_no_client_auth()
};
Ok(builder.with_cert_resolver(resolver))
}
#[cfg(test)]
mod tests {
use super::*;
use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose};
use rustls::{ClientConfig, ClientConnection, RootCertStore, ServerConnection, pki_types::PrivatePkcs8KeyDer};
fn authority(name: &str) -> CertifiedIssuer<'static, KeyPair> {
let mut params = CertificateParams::new(vec![name.to_owned()]).expect("CA names");
params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
CertifiedIssuer::self_signed(params, KeyPair::generate().expect("CA signing key")).expect("self-signed CA")
}
fn handshake(server: ServerConfig, client: ClientConfig) -> io::Result<()> {
let mut server = ServerConnection::new(Arc::new(server)).map_err(io::Error::other)?;
let mut client =
ClientConnection::new(Arc::new(client), "localhost".try_into().expect("server name")).map_err(io::Error::other)?;
for _ in 0..16 {
let mut wire = Vec::new();
client.write_tls(&mut wire)?;
if !wire.is_empty() {
server.read_tls(&mut std::io::Cursor::new(wire))?;
server.process_new_packets().map_err(io::Error::other)?;
}
let mut wire = Vec::new();
server.write_tls(&mut wire)?;
if !wire.is_empty() {
client.read_tls(&mut std::io::Cursor::new(wire))?;
client.process_new_packets().map_err(io::Error::other)?;
}
if !server.is_handshaking() && !client.is_handshaking() {
return Ok(());
}
}
Err(io::Error::other("TLS handshake did not finish"))
}
#[test]
fn protocol_listener_tls_requires_the_configured_client_ca() {
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let directory = tempfile::tempdir().expect("TLS directory");
let trusted = authority("trusted-ca");
let unrelated = authority("unrelated-ca");
let server_key = KeyPair::generate().expect("server key");
let mut params = CertificateParams::new(vec!["localhost".to_owned()]).expect("server names");
params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
let server_cert = params.signed_by(&server_key, &trusted).expect("server certificate");
std::fs::write(directory.path().join("rustfs_cert.pem"), server_cert.pem()).expect("write server certificate");
std::fs::write(directory.path().join("rustfs_key.pem"), server_key.serialize_pem()).expect("write server key");
let ca_file = directory.path().join("clients.pem");
std::fs::write(&ca_file, trusted.pem()).expect("write client CA");
let resolver =
rustfs_tls_runtime::ReloadableServerCertResolver::load_from_directory(directory.path().to_str().expect("directory"))
.expect("production certificate resolver");
for (client_issuer, allowed) in [(None, false), (Some(&unrelated), false), (Some(&trusted), true)] {
let mut roots = RootCertStore::empty();
roots.add(trusted.der().clone()).expect("trust server CA");
let builder = ClientConfig::builder().with_root_certificates(roots);
let client = if let Some(issuer) = client_issuer {
let key = KeyPair::generate().expect("client key");
let mut params = CertificateParams::new(Vec::new()).expect("client names");
params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
let cert = params.signed_by(&key, issuer).expect("client certificate");
builder
.with_client_auth_cert(vec![cert.der().clone()], PrivatePkcs8KeyDer::from(key.serialize_der()).into())
.expect("client identity")
} else {
builder.with_no_client_auth()
};
let server = server_config(resolver.clone(), ca_file.to_str()).expect("production listener TLS policy");
assert_eq!(handshake(server, client).is_ok(), allowed);
}
let mut roots = RootCertStore::empty();
roots.add(trusted.der().clone()).expect("trust server CA");
handshake(
server_config(resolver.clone(), None).expect("ordinary TLS"),
ClientConfig::builder().with_root_certificates(roots).with_no_client_auth(),
)
.expect("no CA preserves ordinary TLS");
for bad in [
"",
"not a certificate",
"-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----",
] {
std::fs::write(&ca_file, bad).expect("bad CA fixture");
assert!(server_config(resolver.clone(), ca_file.to_str()).is_err(), "bad CA must fail startup");
}
std::fs::remove_file(&ca_file).expect("remove CA fixture");
assert!(server_config(resolver, ca_file.to_str()).is_err());
}
#[tokio::test]
async fn protocol_listeners_refuse_startup_with_an_invalid_client_ca() {
let directory = tempfile::tempdir().expect("TLS directory");
let certified = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).expect("server identity");
std::fs::write(directory.path().join("rustfs_cert.pem"), certified.cert.pem()).expect("server certificate");
std::fs::write(directory.path().join("rustfs_key.pem"), certified.signing_key.serialize_pem()).expect("server key");
let ca_file = directory.path().join("clients.pem");
std::fs::write(&ca_file, "not a CA certificate").expect("invalid client CA fixture");
let (_shutdown, shutdown) = tokio::sync::broadcast::channel(1);
#[cfg(feature = "webdav")]
{
let config = crate::webdav::config::WebDavConfig {
bind_addr: "127.0.0.1:0".parse().expect("bind"),
tls_enabled: true,
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
ca_file: Some(ca_file.to_string_lossy().into_owned()),
..Default::default()
};
let server = crate::webdav::server::WebDavServer::new(config, crate::common::dummy_storage::DummyBackend::default())
.await
.expect("path checks");
let error = tokio::time::timeout(std::time::Duration::from_secs(5), server.start(shutdown.resubscribe()))
.await
.expect("startup deadline")
.expect_err("invalid CA must not serve WebDAV");
assert!(matches!(error, crate::webdav::config::WebDavInitError::Tls(_)));
}
#[cfg(feature = "ftps")]
{
let config = crate::ftps::config::FtpsConfig {
bind_addr: "127.0.0.1:0".parse().expect("bind"),
tls_enabled: true,
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
ca_file: Some(ca_file.to_string_lossy().into_owned()),
..Default::default()
};
let server = crate::ftps::server::FtpsServer::new(config, crate::common::dummy_storage::DummyBackend::default())
.await
.expect("path checks");
let error = tokio::time::timeout(std::time::Duration::from_secs(5), server.start(shutdown.resubscribe()))
.await
.expect("startup deadline")
.expect_err("invalid CA must not serve FTPS");
assert!(matches!(error, crate::ftps::config::FtpsInitError::InvalidConfig(_)));
}
}
#[tokio::test]
async fn protocol_client_ca_configuration_cannot_allow_cleartext() {
let directory = tempfile::tempdir().expect("config directory");
let ca_file = directory.path().join("clients.pem");
std::fs::write(&ca_file, "test").expect("existing CA path");
#[cfg(feature = "webdav")]
{
let config = crate::webdav::config::WebDavConfig {
tls_enabled: false,
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
ca_file: Some(ca_file.to_string_lossy().into_owned()),
..Default::default()
};
assert!(config.validate().await.is_err());
}
#[cfg(feature = "ftps")]
{
let config = crate::ftps::config::FtpsConfig {
tls_enabled: false,
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
ca_file: Some(ca_file.to_string_lossy().into_owned()),
..Default::default()
};
assert!(config.validate().await.is_err());
}
}
}
-6
View File
@@ -56,12 +56,6 @@ impl FtpsConfig {
));
}
if self.ca_file.is_some() && (!self.tls_enabled || self.cert_dir.is_none()) {
return Err(FtpsInitError::InvalidConfig(
"A client CA requires TLS and a server certificate directory".to_owned(),
));
}
if let Some(path) = &self.cert_dir
&& !tokio::fs::try_exists(path).await.unwrap_or(false)
{
+9 -8
View File
@@ -110,9 +110,6 @@ where
ftps_required = self.config.ftps_required,
"FTPS server starting"
);
if self.config.tls_enabled && self.config.cert_dir.is_some() {
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
}
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
let storage_clone = self.storage.clone();
@@ -121,7 +118,7 @@ where
Arc::new(ConnectionUserDetailProvider),
)
.authenticator(Arc::new(ConnectionAuthenticator {
require_data_tls: self.config.tls_enabled && (self.config.ftps_required || self.config.ca_file.is_some()),
require_data_tls: self.config.tls_enabled && self.config.ftps_required,
}));
// Configure passive ports for data connections
@@ -193,12 +190,16 @@ where
reload_shutdown_rx.clone(),
);
let server_config = crate::common::tls::server_config(resolver, self.config.ca_file.as_deref())
.map_err(|error| FtpsInitError::InvalidConfig(format!("Invalid client CA configuration: {error}")))?;
// Build ServerConfig with SNI support
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let server_config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(resolver);
server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config));
if self.config.ftps_required || self.config.ca_file.is_some() {
if self.config.ftps_required {
debug!(
event = EVENT_FTPS_TLS_STATE,
component = LOG_COMPONENT_PROTOCOLS,
@@ -208,7 +209,7 @@ where
);
server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All);
}
} else if self.config.ftps_required || self.config.ca_file.is_some() {
} else if self.config.ftps_required {
return Err(FtpsInitError::InvalidConfig(
"FTPS required but certificate directory not provided".into(),
));
-67
View File
@@ -43,70 +43,3 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
#[cfg(feature = "sftp")]
pub use sftp::{SftpConfig, SftpInitError, SftpServer};
#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
mod fips_tests {
use crate::common::dummy_storage::DummyBackend;
use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
#[test]
fn protocols_reject_incompatible_host_provider() {
const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
if std::env::var_os(CHILD).is_some() {
let mut provider = rustfs_tls_runtime::default_crypto_provider();
provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
provider.install_default().expect("install incompatible host provider");
let cert_dir = tempfile::tempdir().expect("certificate directory");
let cert_dir = cert_dir.path().to_string_lossy().into_owned();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("test runtime");
runtime.block_on(async {
let ftps_config = FtpsConfig {
cert_dir: Some(cert_dir.clone()),
..Default::default()
};
let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
.await
.expect("FTPS config");
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
let error = ftps
.start(receiver)
.await
.expect_err("FTPS must reject the incompatible provider");
assert!(error.to_string().contains("process-wide rustls crypto provider"));
let webdav_config = WebDavConfig {
bind_addr: ([127, 0, 0, 1], 0).into(),
cert_dir: Some(cert_dir),
..Default::default()
};
let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
.await
.expect("WebDAV config");
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
let error = webdav
.start(receiver)
.await
.expect_err("WebDAV must reject the incompatible provider");
assert!(error.to_string().contains("process-wide rustls crypto provider"));
});
return;
}
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
.arg("protocols_reject_incompatible_host_provider")
.arg("--nocapture")
.env(CHILD, "1")
.env("NO_PROXY", "localhost,127.0.0.1,::1")
.env("no_proxy", "localhost,127.0.0.1,::1")
.output()
.expect("start isolated protocol TLS test");
assert!(
output.status.success(),
"stdout={} stderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
}
-6
View File
@@ -65,12 +65,6 @@ impl WebDavConfig {
));
}
if self.ca_file.is_some() && (!self.tls_enabled || self.cert_dir.is_none()) {
return Err(WebDavInitError::InvalidConfig(
"A client CA requires TLS and a server certificate directory".to_owned(),
));
}
if let Some(path) = &self.cert_dir
&& !tokio::fs::try_exists(path).await.unwrap_or(false)
{
+4 -6
View File
@@ -30,6 +30,7 @@ use hyper_util::rt::{TokioIo, TokioTimer};
use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
use rustfs_utils::MaskedAccessKey;
use rustls::ServerConfig;
use std::convert::Infallible;
use std::io;
use std::net::IpAddr;
@@ -117,10 +118,6 @@ where
"WebDAV server starting"
);
if self.config.tls_enabled && self.config.cert_dir.is_some() {
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
}
let listener = TcpListener::bind(self.config.bind_addr).await?;
self.serve(listener, shutdown_rx).await
}
@@ -158,8 +155,9 @@ where
reload_shutdown_rx.clone(),
);
let server_config = crate::common::tls::server_config(resolver, self.config.ca_file.as_deref())
.map_err(|error| WebDavInitError::Tls(format!("Invalid client CA configuration: {error}")))?;
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
Some(TlsAcceptor::from(Arc::new(server_config)))
} else {

Some files were not shown because too many files have changed in this diff Show More