mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-04 18:13:57 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a5e2aab1b4 | |||
| ae6fa93ac3 | |||
| c830af66d5 | |||
| 5345772af4 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=16b3cd69887a73b468c84012bf5c0cb7e24360fcaf6ab3c13967020e37ef86d2
|
||||
sha256-linux=c7c7f0c7ce2610bc0a86a518bfa67a3ed56d862aa44f05b4a92200fabf51d6c7
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
|
||||
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
|
||||
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
|
||||
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
|
||||
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
|
||||
@@ -32,8 +32,6 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -769,10 +769,12 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -847,11 +849,12 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual",
|
||||
"release"
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -497,63 +497,6 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
@@ -1454,7 +1397,6 @@ jobs:
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1482,7 +1424,6 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,6 +33,32 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,19 +12,30 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -33,7 +44,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -87,206 +87,18 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Packaging tools are off on this lane, so setup does not install unzip.
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq unzip
|
||||
fi
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "unzip is required to extract console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p ./rustfs/static
|
||||
|
||||
verify_sha256() {
|
||||
local expected="$1"
|
||||
local file="$2"
|
||||
local actual=""
|
||||
local actual_line=""
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual_line=$(sha256sum -- "$file") || return 1
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual_line=$(shasum -a 256 -- "$file") || return 1
|
||||
else
|
||||
echo "No SHA-256 verification tool found" >&2
|
||||
return 1
|
||||
fi
|
||||
actual="${actual_line%%[[:space:]]*}"
|
||||
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
echo "SHA256 verified OK: $actual"
|
||||
return 0
|
||||
else
|
||||
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
local curl_auth_args=()
|
||||
console_tag=""
|
||||
|
||||
chmod +w ./rustfs/static/LICENSE || true
|
||||
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin="python"
|
||||
fi
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
echo "No Python interpreter found for release metadata parsing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
||||
fi
|
||||
|
||||
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
||||
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
|
||||
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("no console zip asset with sha256 digest found")
|
||||
PY
|
||||
) || return 1
|
||||
|
||||
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
||||
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Console release: ${console_tag}"
|
||||
echo "Downloading console asset: ${console_url}"
|
||||
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
verify_sha256 "$console_sha256" console.zip || return 2
|
||||
unzip -o console.zip -d ./rustfs/static || return 2
|
||||
}
|
||||
|
||||
if download_console_assets; then
|
||||
rm -f console.zip console-release.json
|
||||
else
|
||||
status=$?
|
||||
rm -f console.zip console-release.json
|
||||
if [[ "$status" -eq 2 ]]; then
|
||||
echo "Console asset integrity verification failed" >&2
|
||||
fi
|
||||
echo "Failed to download verified console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s ./rustfs/static/index.html ]]; then
|
||||
echo "Console asset archive is missing static/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
||||
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
||||
|
||||
- name: Build RustFS
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
shell: bash
|
||||
run: |
|
||||
# Force rebuild by touching build.rs so a restored cache cannot
|
||||
# reuse a rustfs artifact compiled before these console assets existed.
|
||||
touch rustfs/build.rs
|
||||
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Verify console
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
data_dir="$(mktemp -d)"
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$data_dir"
|
||||
' EXIT
|
||||
|
||||
if [[ ! -x "$binary" ]]; then
|
||||
echo "rustfs binary not found: ${binary}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_port="$(find_free_port)"
|
||||
console_port="$(find_free_port)"
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$binary" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
response=""
|
||||
for _ in {1..40}; do
|
||||
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "Console endpoint did not return 200 text/html: $response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
@@ -534,12 +346,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -548,90 +360,6 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
@@ -92,17 +92,6 @@ jobs:
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
@@ -110,15 +99,15 @@ jobs:
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = candidate["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
@@ -132,26 +121,6 @@ jobs:
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
@@ -255,8 +224,7 @@ jobs:
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
@@ -275,7 +243,6 @@ jobs:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
@@ -384,10 +351,7 @@ jobs:
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
|
||||
@@ -107,6 +107,32 @@ jobs:
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
|
||||
|
||||
# Native multipart completion publishes its payload from a different
|
||||
# parent than its metadata, and abort renames under a precondition; both
|
||||
# need their own Windows handle discipline.
|
||||
- name: Test native multipart complete and abort publication
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
|
||||
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
|
||||
--nocapture
|
||||
|
||||
# Owned-undo reclaim removes a directory it pinned, copy heal persists
|
||||
# fault bytes through a handle, and metadata reads meet a directory.
|
||||
- name: Test native multipart reclaim, copy heal, and directory reads
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_multipart_owned_undo
|
||||
native_multipart_copy_heal
|
||||
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
|
||||
--nocapture
|
||||
|
||||
- name: Test directory removal fallback
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
|
||||
|
||||
- name: Test Windows path conversion
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+195
-377
File diff suppressed because it is too large
Load Diff
+17
-14
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -212,13 +212,12 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.19.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -235,7 +234,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.39" }
|
||||
jiff = { version = "0.2.38" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -254,9 +253,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -338,7 +337,7 @@ strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
test-case = "3.4.0"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -357,13 +356,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.3.0"
|
||||
zip = "9.1.3"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -378,16 +377,16 @@ libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.1" }
|
||||
russh = { version = "0.65.0" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
hotpath = { version = "0.30.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -428,3 +427,7 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
|
||||
/// Default validation mode is "hop_by_hop".
|
||||
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
|
||||
|
||||
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
|
||||
/// Environment variable to enable RFC 7239 "Forwarded" header support.
|
||||
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
|
||||
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
|
||||
/// RFC 7239 support is enabled by default.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
|
||||
|
||||
/// Environment variable for the maximum allowed proxy hops.
|
||||
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
|
||||
|
||||
@@ -146,9 +146,3 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
|
||||
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
|
||||
|
||||
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
|
||||
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -12,249 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ mod tests {
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -1654,89 +1654,6 @@ mod tests {
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn release_heal_commit_barrier_after_shutdown(
|
||||
log: &Path,
|
||||
log_offset: u64,
|
||||
barrier: &Path,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
// Observe within the existing graceful process-stop budget.
|
||||
timeout(Duration::from_secs(45), async {
|
||||
loop {
|
||||
let mut file = std::fs::File::open(log)?;
|
||||
file.seek(SeekFrom::Start(log_offset))?;
|
||||
let mut tail = String::new();
|
||||
file.read_to_string(&mut tail)?;
|
||||
if let Some(event) = tail
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|event| {
|
||||
event["event"] == "server_shutdown_state"
|
||||
&& event["component"] == "main"
|
||||
&& event["subsystem"] == "startup"
|
||||
&& event["state"] == "stopping"
|
||||
})
|
||||
{
|
||||
// Local heal has stopped and HTTP shutdown has been signalled.
|
||||
// Let admitted peer RPCs finish; keeping this synthetic barrier
|
||||
// until process exit would make the fixture block its own drain.
|
||||
std::fs::remove_file(barrier)?;
|
||||
return Ok(event);
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
let event = serde_json::json!({
|
||||
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
|
||||
});
|
||||
std::fs::write(&log, format!("{event}\n"))?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let log_offset = log.metadata()?.len();
|
||||
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
|
||||
|
||||
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
|
||||
tokio::pin!(release);
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"old or unrelated events must not release the barrier"
|
||||
);
|
||||
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"background shutdown may take longer than five seconds within the process-stop budget"
|
||||
);
|
||||
writeln!(writer, "{event}")?;
|
||||
tokio::time::advance(Duration::from_millis(10)).await;
|
||||
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
|
||||
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
std::fs::write(&log, "")?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
|
||||
.await
|
||||
.expect_err("missing shutdown evidence must fail the fixture");
|
||||
assert!(error.is::<tokio::time::error::Elapsed>());
|
||||
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_case = match scenario {
|
||||
@@ -1874,7 +1791,6 @@ mod tests {
|
||||
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
let mut shutdown_barrier_release = None;
|
||||
let mut attempt_count = 0usize;
|
||||
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||
while expected_manifests.len() < online_object_count {
|
||||
@@ -2457,21 +2373,7 @@ mod tests {
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
) {
|
||||
if interruption_node == 1
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
|
||||
let log_offset = target_log.metadata()?.len();
|
||||
let (stopped, released) = tokio::join!(
|
||||
cluster.stop_node_gracefully(interruption_node),
|
||||
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
|
||||
);
|
||||
// Await both branches even on error so the child is reaped.
|
||||
stopped?;
|
||||
shutdown_barrier_release = Some(released?);
|
||||
} else {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
}
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
}
|
||||
@@ -2513,9 +2415,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
if shutdown_barrier_release.is_none()
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
std::fs::remove_file(barrier)?;
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
@@ -2846,7 +2746,6 @@ mod tests {
|
||||
"all_peers_bound_to_generation_window": true,
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"graceful_barrier_release": shutdown_barrier_release,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
if let Some(outage_write) = outage_write_diagnostic {
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -33,7 +33,6 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
@@ -117,10 +116,6 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -3511,8 +3421,6 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3580,24 +3488,6 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3661,16 +3551,6 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3792,12 +3672,7 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
Err(EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -4296,8 +4171,6 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4348,103 +4221,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5528,8 +5304,6 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6494,108 +6268,6 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
source_already_purged
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
|
||||
@@ -32,6 +32,7 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -46,7 +47,6 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,6 +58,8 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
|
||||
@@ -1637,7 +1637,6 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1709,7 +1708,6 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1780,7 +1778,6 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2012,7 +2009,6 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2027,7 +2023,6 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -5745,7 +5740,6 @@ mod tests {
|
||||
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
fail_format_reads: Arc<AtomicBool>,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
@@ -5765,7 +5759,6 @@ mod tests {
|
||||
object_read_all_disks: Arc::default(),
|
||||
format_data,
|
||||
read_all_data,
|
||||
fail_format_reads: Arc::default(),
|
||||
next_format_read_gate: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
@@ -5889,9 +5882,6 @@ mod tests {
|
||||
gate.entered.notify_one();
|
||||
gate.release.cancelled().await;
|
||||
}
|
||||
if peer.fail_format_reads.load(Ordering::Acquire) {
|
||||
return Err(Status::unavailable("format read temporarily unavailable"));
|
||||
}
|
||||
peer.format_data.clone()
|
||||
} else {
|
||||
peer.object_read_all_disks
|
||||
@@ -8857,124 +8847,6 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
|
||||
(rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
|
||||
],
|
||||
async {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(reference.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"recovered-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let disk = crate::disk::new_disk(
|
||||
&set.set_endpoints[0],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("registered remote disk should construct");
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
panic!("fixture must create a remote disk");
|
||||
};
|
||||
assert!(remote.health_check);
|
||||
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
set.disks.write().await[0] = Some(Arc::clone(&disk));
|
||||
|
||||
peer.peer.fail_format_reads.store(true, Ordering::Release);
|
||||
set.connect_disks().await;
|
||||
peer.peer.fail_format_reads.store(false, Ordering::Release);
|
||||
let cancelled_after_failure = remote.cancel_token.is_cancelled();
|
||||
assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
|
||||
assert!(
|
||||
Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
|
||||
"a failed reconnect must not publish its unvalidated candidate"
|
||||
);
|
||||
remote.spawn_recovery_monitor_if_needed();
|
||||
let recovery = time::timeout(Duration::from_secs(5), async {
|
||||
while !disk.is_online().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
disk.read_all("bucket", "object").await
|
||||
})
|
||||
.await;
|
||||
disk.close().await.expect("stop the registered disk's monitoring");
|
||||
peer.stop().await;
|
||||
assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
|
||||
assert_eq!(
|
||||
recovery
|
||||
.expect("failed reconnect must leave the registered disk able to recover without another sweep")
|
||||
.expect("the recovered registered disk must serve real I/O"),
|
||||
Bytes::from_static(b"recovered-peer-data")
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(reference.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"replacement-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let disk = crate::disk::new_disk(
|
||||
&set.set_endpoints[0],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("registered remote disk should construct");
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
panic!("fixture must create a remote disk");
|
||||
};
|
||||
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
set.disks.write().await[0] = Some(Arc::clone(&disk));
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
|
||||
time::timeout(Duration::from_secs(5), gate.entered.notified())
|
||||
.await
|
||||
.expect("replacement must reach the format validation barrier");
|
||||
let cancelled_before_publication = remote.cancel_token.is_cancelled();
|
||||
gate.release.cancel();
|
||||
pending.await.expect("validated replacement must finish publication");
|
||||
let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
|
||||
let readback = replacement.read_all("bucket", "object").await;
|
||||
replacement.close().await.expect("stop replacement monitoring");
|
||||
peer.stop().await;
|
||||
assert!(
|
||||
!cancelled_before_publication,
|
||||
"the registered monitor must survive pending format validation"
|
||||
);
|
||||
assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
|
||||
assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
|
||||
assert_eq!(
|
||||
readback.expect("replacement must serve real I/O"),
|
||||
Bytes::from_static(b"replacement-peer-data")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
|
||||
|
||||
@@ -445,10 +445,6 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
});
|
||||
// A successful lock RPC over a cached or fresh channel proves the peer is
|
||||
// reachable; reconcile the internode peer-health gauge so it does not stay
|
||||
// stale when no fresh dial fires (issue #9119).
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
@@ -676,7 +672,6 @@ impl RemoteClient {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
late(response).await;
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -629,559 +628,3 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -94,6 +94,7 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -6838,47 +6839,11 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7269,8 +7234,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7325,8 +7288,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7340,10 +7301,6 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7357,36 +7314,19 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, selected_raw, mut selected) =
|
||||
let (_, _, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -8380,9 +8320,8 @@ where
|
||||
})
|
||||
});
|
||||
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
|
||||
let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
|
||||
let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
|
||||
if matches!(&result, Err(Error::PreconditionFailed)) {
|
||||
let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
|
||||
if !may_have_mutated {
|
||||
record_pool_meta_stale_write_rejection(phase);
|
||||
}
|
||||
let mut source = None;
|
||||
@@ -8393,8 +8332,6 @@ where
|
||||
source = Some(Arc::clone(&original));
|
||||
if matches!(original.as_ref(), Error::PreconditionFailed) {
|
||||
Err(Error::PreconditionFailed)
|
||||
} else if write_not_started {
|
||||
Err(Error::other(original))
|
||||
} else {
|
||||
Err(Error::other(pool_metadata_error(
|
||||
crate::error::PoolMetadataFailure::TransactionUnknown,
|
||||
@@ -14609,23 +14546,6 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14636,10 +14556,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != repair_data {
|
||||
if confirmed != *canonical {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -29562,53 +29482,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
@@ -30829,154 +30702,6 @@ mod pools_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
|
||||
let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let data =
|
||||
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
|
||||
let mut transaction = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("two of four disks cannot satisfy the identity write quorum");
|
||||
drop(transaction);
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(crate::disk::RUSTFS_META_BUCKET)
|
||||
.join(POOL_META_IDENTITY_NAME)
|
||||
.exists(),
|
||||
"the rejected write must not create an identity replica"
|
||||
);
|
||||
}
|
||||
state
|
||||
.ensure_write_safe("retry after disks return")
|
||||
.expect("a rejected inline write must not poison the startup transaction");
|
||||
|
||||
let ordinary_error = crate::config::com::save_config_with_opts(
|
||||
pool.clone(),
|
||||
"ordinary-config",
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("ordinary writes must still fail without quorum");
|
||||
assert!(
|
||||
matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
|
||||
"ordinary callers must retain their existing quorum error"
|
||||
);
|
||||
|
||||
*pool.disks.write().await = disks.into_iter().map(Some).collect();
|
||||
let mut retry = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut retry,
|
||||
)
|
||||
.await
|
||||
.expect("the same transaction must succeed when the disks return");
|
||||
retry.disarm();
|
||||
drop(retry);
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
|
||||
.await
|
||||
.expect("read back the committed identity"),
|
||||
data
|
||||
);
|
||||
state
|
||||
.ensure_write_safe("after successful retry")
|
||||
.expect("retry must leave writes enabled");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
|
||||
let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let data =
|
||||
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
|
||||
let mut transaction = state.arm_transaction();
|
||||
let written = save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"prepare_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect("the earlier phase must really commit");
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("the next phase must fail without a write quorum");
|
||||
drop(transaction);
|
||||
*pool.disks.write().await = disks.into_iter().map(Some).collect();
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
|
||||
.await
|
||||
.expect("the earlier commit must survive the rejected write"),
|
||||
data
|
||||
);
|
||||
let error = state
|
||||
.ensure_write_safe("after a committed phase")
|
||||
.expect_err("a later rejection cannot discharge the earlier write");
|
||||
assert_eq!(
|
||||
error.pool_metadata_failure().expect("retain typed uncertainty").kind,
|
||||
crate::error::PoolMetadataFailure::TransactionUnknown
|
||||
);
|
||||
assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
|
||||
let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let mut transaction = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool,
|
||||
POOL_META_IDENTITY_NAME,
|
||||
vec![0x51; 2 * 1024 * 1024],
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("the noninline write must fail without quorum");
|
||||
drop(transaction);
|
||||
assert!(
|
||||
state.ensure_write_safe("after writer setup").is_err(),
|
||||
"a failure after disk writers were created must remain fail-closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum IdentityRepairFault {
|
||||
Healthy,
|
||||
|
||||
@@ -853,8 +853,6 @@ pub(crate) struct DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
|
||||
/// Caller API operations currently executing through the disk health wrapper.
|
||||
api_waiting: AtomicU32,
|
||||
/// Operations rejected because the disk was unavailable for the caller.
|
||||
@@ -882,8 +880,6 @@ impl Default for DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex::new(None),
|
||||
api_waiting: AtomicU32::new(0),
|
||||
total_errors_availability: AtomicU64::new(0),
|
||||
total_errors_timeout: AtomicU64::new(0),
|
||||
@@ -1577,20 +1573,6 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScopedDiskOperationTimeout {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
previous: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedDiskOperationTimeout {
|
||||
fn drop(&mut self) {
|
||||
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1932,23 +1914,6 @@ impl LocalDiskWrapper {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
|
||||
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
|
||||
// The metrics epoch follows this drive across clones and reconnects.
|
||||
// Its scoped override cannot shorten unrelated healthy drives' I/O.
|
||||
let previous = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.replace(timeout);
|
||||
ScopedDiskOperationTimeout {
|
||||
metrics: Arc::clone(&self.metrics),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -2673,13 +2638,6 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#[cfg(test)]
|
||||
let timeout_duration = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
@@ -6086,67 +6044,6 @@ mod tests {
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
|
||||
let directories = [
|
||||
tempfile::tempdir().expect("faulted drive"),
|
||||
tempfile::tempdir().expect("healthy drive"),
|
||||
];
|
||||
let mut wrappers = Vec::new();
|
||||
for directory in &directories {
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
|
||||
wrappers.push(LocalDiskWrapper::new(disk, false));
|
||||
}
|
||||
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::clone(&wrappers[0].disk),
|
||||
false,
|
||||
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
|
||||
);
|
||||
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
|
||||
tokio::time::pause();
|
||||
let operation = || async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(())
|
||||
};
|
||||
let (faulted, healthy) = tokio::join!(
|
||||
reconnected.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
wrappers[1].track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
);
|
||||
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
|
||||
healthy.expect("healthy drive keeps its caller's deadline");
|
||||
let shorter_started = tokio::time::Instant::now();
|
||||
let shorter = wrappers[0]
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_millis(500),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
|
||||
assert!(shorter_started.elapsed() < Duration::from_secs(1));
|
||||
wrappers[0]
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
|
||||
.await
|
||||
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
|
||||
drop(scope);
|
||||
reconnected
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
|
||||
.await
|
||||
.expect("dropping the scope restores reconnected callers' deadlines");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ignored_timeout_does_not_mark_drive_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -5,7 +5,6 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -53,11 +52,7 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -130,7 +125,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
"new entries must survive the earlier volume predicate"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -169,7 +164,6 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
@@ -183,459 +177,3 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
assert_eq!(fresh.len, 11);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
|
||||
let (_directory, file) = fixture();
|
||||
let mut cache = FdCache::new();
|
||||
|
||||
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
|
||||
cache.invalidate_under("v", "missing/prefix");
|
||||
cache.invalidate_volume("v");
|
||||
cache.invalidate_exact("v", "missing/prefix/part.1").await;
|
||||
cache.clear();
|
||||
assert!(
|
||||
cache.maintenance_task.get().is_none(),
|
||||
"empty-cache invalidations must not spawn maintenance"
|
||||
);
|
||||
|
||||
// Populate a normal entry and verify matched vs unmatched filter gates directly,
|
||||
// and ensure unmatched invalidations do not enqueue pending removals.
|
||||
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(cache.tracker.live_entries(), 1);
|
||||
assert!(cache.tracker.may_match_volume("v"));
|
||||
assert!(!cache.tracker.may_match_volume("other-volume"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
|
||||
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
|
||||
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
|
||||
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
|
||||
|
||||
cache.invalidate_volume("other-volume");
|
||||
cache.invalidate_under("v", "obj-2/0198a3f0");
|
||||
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
|
||||
assert!(
|
||||
!cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"unmatched invalidations must not enqueue maintenance removals"
|
||||
);
|
||||
assert!(cache.get(&cached_key).await.is_some());
|
||||
|
||||
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
|
||||
// to per-volume unindexed_counts without degrading other volumes.
|
||||
let other_vol_key = key("v2", "shallow/part.1", false);
|
||||
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
|
||||
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
|
||||
let deep_key = key("v", deep_path, false);
|
||||
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
|
||||
"deep key in volume v must not disable prefix filtering in volume v2"
|
||||
);
|
||||
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
|
||||
assert!(cache.get(&deep_key).await.is_none());
|
||||
cache.invalidate_volume("v2");
|
||||
assert!(cache.get(&other_vol_key).await.is_none());
|
||||
|
||||
// Replacing an invalidated key before maintenance drains pending_invalidations
|
||||
// must release the old descriptor immediately and keep the replacement live
|
||||
// even when a strong reference to the old CachedFdEntry shell survives until
|
||||
// maintenance runs its Arc::ptr_eq check.
|
||||
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
|
||||
let old_weak = Arc::downgrade(&old_file);
|
||||
cache.insert(cached_key.clone(), old_file).await;
|
||||
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
|
||||
cache.invalidate_under("v", "obj-1/0198a3f0");
|
||||
assert!(cache.get(&cached_key).await.is_none());
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
old_weak.upgrade().is_none(),
|
||||
"replacing an entry must retire and drop the previous descriptor immediately"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("maintenance should drain pending invalidations");
|
||||
drop(held_old_shell);
|
||||
assert!(
|
||||
cache.get(&cached_key).await.is_some(),
|
||||
"stale pending invalidation must not evict a newer replacement"
|
||||
);
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
0
|
||||
);
|
||||
|
||||
// Dropping a cache while a live non-invalidated entry is present (even if a
|
||||
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
|
||||
// must retire the descriptor and release FdCacheTracker without cycles.
|
||||
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
|
||||
let live_drop_weak = Arc::downgrade(&live_drop_file);
|
||||
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
|
||||
let shutdown_weak = Arc::downgrade(&shutdown_file);
|
||||
let tracker_weak = Arc::downgrade(&cache.tracker);
|
||||
let cloned_moka = cache.cache.clone();
|
||||
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
|
||||
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
|
||||
cache.invalidate_under("v", "shutdown");
|
||||
if let Some(handle) = cache.maintenance_task.take() {
|
||||
handle.abort();
|
||||
let _ = handle.await;
|
||||
}
|
||||
drop(cache);
|
||||
assert!(
|
||||
live_drop_weak.upgrade().is_none(),
|
||||
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
|
||||
);
|
||||
assert!(
|
||||
shutdown_weak.upgrade().is_none(),
|
||||
"dropping cache when maintenance is aborted must still release descriptors"
|
||||
);
|
||||
assert!(
|
||||
tracker_weak.upgrade().is_none(),
|
||||
"dropping cache must release FdCacheTracker without reference cycles"
|
||||
);
|
||||
drop(cloned_moka);
|
||||
|
||||
// Short-TTL expired entries (including mixed expired and fresh entries) must
|
||||
// be retired by invalidate_under and clear() even though Moka's iterator skips
|
||||
// expired entries.
|
||||
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
|
||||
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
|
||||
let expired_weak = Arc::downgrade(&expired_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
|
||||
let fresh_weak = Arc::downgrade(&fresh_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
|
||||
ttl_cache.invalidate_under("v", "ttl-obj");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("invalidate_under must retire both expired and fresh matching descriptors");
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
|
||||
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
|
||||
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
|
||||
ttl_cache
|
||||
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
|
||||
.await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
|
||||
let clear_live_weak = Arc::downgrade(&clear_live_file);
|
||||
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
|
||||
ttl_cache.clear();
|
||||
assert!(
|
||||
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
|
||||
"clear() must synchronously retire both TTL-expired and live descriptors"
|
||||
);
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
|
||||
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
|
||||
|
||||
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
|
||||
let paths = [
|
||||
"a/b",
|
||||
"a/b/part.1",
|
||||
"a/b/dir/part.2",
|
||||
"a/bc",
|
||||
"a/bc/part.1",
|
||||
"prefix/obj-0001.bin/0198a3f0/part.1",
|
||||
"prefix/obj-0002.bin/0198a3f0/part.1",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
|
||||
];
|
||||
let prefixes = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/",
|
||||
"a/b/dir",
|
||||
"a/bc",
|
||||
"prefix",
|
||||
"prefix/obj-0001.bin",
|
||||
"prefix/obj-0001.bin/0198a3f0",
|
||||
"d1/d2/d3/d4/d5/c_missing",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
|
||||
"nonexistent/prefix",
|
||||
];
|
||||
|
||||
let mut candidate_keys = Vec::new();
|
||||
for v in volumes {
|
||||
for p in paths {
|
||||
for direct in [false, true] {
|
||||
candidate_keys.push(key(v, p, direct));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut next_u64 = || {
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 7;
|
||||
rng ^= rng << 17;
|
||||
rng
|
||||
};
|
||||
|
||||
for step in 0..1000 {
|
||||
let op = next_u64() % 100;
|
||||
if op < 45 {
|
||||
// Insert or replace (either fresh or stale generation)
|
||||
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: step,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let cur_gen = cache.generation();
|
||||
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
|
||||
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
|
||||
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
|
||||
if !stale {
|
||||
oracle.insert(k, entry);
|
||||
}
|
||||
} else if op < 65 {
|
||||
// Exact invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let p = paths[(next_u64() as usize) % paths.len()];
|
||||
cache.invalidate_exact(v, p).await;
|
||||
oracle.remove(&key(v, p, false));
|
||||
oracle.remove(&key(v, p, true));
|
||||
} else if op < 88 {
|
||||
// Prefix invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
|
||||
cache.invalidate_under(v, raw_prefix);
|
||||
let trimmed = raw_prefix.trim_end_matches('/');
|
||||
oracle.retain(|k, _| {
|
||||
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
|
||||
});
|
||||
} else if op < 97 {
|
||||
// Volume invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
cache.invalidate_volume(v);
|
||||
oracle.retain(|k, _| k.volume != v);
|
||||
} else {
|
||||
// Full clear
|
||||
cache.clear();
|
||||
oracle.clear();
|
||||
}
|
||||
|
||||
// Verify lockstep equivalence against the reference oracle after every operation.
|
||||
for k in &candidate_keys {
|
||||
let got = cache.get(k).await;
|
||||
let expected = oracle.get(k);
|
||||
match (got, expected) {
|
||||
(Some(g), Some(e)) => assert!(
|
||||
Arc::ptr_eq(&g, e),
|
||||
"step {step}: pointer mismatch for key ({}, {}, {})",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct
|
||||
),
|
||||
(None, None) => {}
|
||||
(g, e) => panic!(
|
||||
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct,
|
||||
g.is_some(),
|
||||
e.is_some()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear must drain all live entries and pending invalidations");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = Arc::new(FdCache::new());
|
||||
|
||||
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
|
||||
// at `entry_shards[shard].lock()` after it has passed the initial generation
|
||||
// check and incremented `volume_counts`, bump generation via an unmatched
|
||||
// invalidation, and verify the post-`new()` recheck refuses publication and
|
||||
// preserves the existing entry.
|
||||
let seam_key = key("v-seam", "obj-1/part.1", false);
|
||||
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
|
||||
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
|
||||
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
|
||||
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
|
||||
let stale_gen = cache.generation();
|
||||
let stale_entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: 999_999,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let blocked_insert = {
|
||||
let hold = cache.tracker.entry_shards[entry_shard].lock();
|
||||
let cache_clone = Arc::clone(&cache);
|
||||
let seam_key_clone = seam_key.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
|
||||
});
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"insert never reached CachedFdEntry::new with the previous entry still live"
|
||||
);
|
||||
std::thread::yield_now();
|
||||
}
|
||||
cache.invalidate_under("v-seam", "unmatched-prefix");
|
||||
drop(hold);
|
||||
task
|
||||
};
|
||||
blocked_insert.await.expect("blocked insert task should complete");
|
||||
let preserved = cache
|
||||
.get(&seam_key)
|
||||
.await
|
||||
.expect("refused stale insert must not retire a valid previous entry");
|
||||
assert!(Arc::ptr_eq(&preserved, &initial_entry));
|
||||
cache.invalidate_volume("v-seam");
|
||||
|
||||
let floor_v1 = Arc::new(AtomicU64::new(0));
|
||||
let floor_v2 = Arc::new(AtomicU64::new(0));
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for t in 0..8usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let file = Arc::clone(&file);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
let k = key(vol, &path, i.is_multiple_of(2));
|
||||
let g = cache.generation();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: g,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
cache.insert_if_fresh(k.clone(), entry, g).await;
|
||||
let min_gen = if is_v1 {
|
||||
floor_v1.load(Ordering::SeqCst)
|
||||
} else {
|
||||
floor_v2.load(Ordering::SeqCst)
|
||||
};
|
||||
if let Some(hit) = cache.get(&k).await {
|
||||
assert!(
|
||||
hit.len >= min_gen,
|
||||
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
|
||||
hit.len
|
||||
);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for t in 0..4usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
match i % 4 {
|
||||
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
|
||||
1 => {
|
||||
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
cache.invalidate_exact(vol, &p).await;
|
||||
}
|
||||
2 => {
|
||||
let before = cache.generation();
|
||||
cache.invalidate_volume(vol);
|
||||
let floor = before + 1;
|
||||
if is_v1 {
|
||||
floor_v1.fetch_max(floor, Ordering::SeqCst);
|
||||
} else {
|
||||
floor_v2.fetch_max(floor, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.await.expect("worker task should succeed");
|
||||
}
|
||||
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear after concurrent stress must drain all live entries");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
native_io(move || create_dir_all_std(&path)).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
fn is_dir_error(path: &Path, e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
// macOS: remove_file on a directory returns EPERM
|
||||
|| (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
|
||||
|| is_windows_dir_access_denied(path, e)
|
||||
}
|
||||
|
||||
/// Windows reports `remove_file` on a directory (or directory symlink) as
|
||||
/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
|
||||
/// failure without inspecting the entry itself.
|
||||
#[cfg(windows)]
|
||||
fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
|
||||
use std::os::windows::fs::FileTypeExt;
|
||||
|
||||
e.kind() == io::ErrorKind::PermissionDenied
|
||||
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| {
|
||||
let file_type = metadata.file_type();
|
||||
file_type.is_dir() || file_type.is_symlink_dir()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || remove_std(path)).await
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || match std::fs::remove_file(&path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir_all(path)).await
|
||||
}
|
||||
Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
match std::fs::remove_file(path.as_ref()) {
|
||||
let path = path.as_ref();
|
||||
match std::fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
|
||||
Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -198,7 +198,12 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
// Windows only flushes a handle opened for writing.
|
||||
#[cfg(windows)]
|
||||
let backup = std::fs::OpenOptions::new().write(true).open(&backup_path)?;
|
||||
#[cfg(not(windows))]
|
||||
let backup = std::fs::File::open(&backup_path)?;
|
||||
backup.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -1594,12 +1599,13 @@ impl LocalDisk {
|
||||
);
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
let payload_guard = if multipart_publication.is_some() {
|
||||
Some(
|
||||
os::multipart_payload_commit_guard(
|
||||
&commit.directory_guard,
|
||||
src_data_path.parent().ok_or(DiskError::InvalidPath)?,
|
||||
commit.publication_root,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1608,9 +1614,9 @@ impl LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
let directory_guard = &commit.directory_guard;
|
||||
if let Err(err) = os::rename_all_with_commit_guard(
|
||||
src_data_path,
|
||||
|
||||
@@ -1002,17 +1002,21 @@ pub(super) fn inspection_with_markers(
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
|
||||
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.remove(rollback)
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
|
||||
@@ -270,17 +270,21 @@ struct ActiveMultipartSourceEntry {
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
|
||||
let source = source.as_ref();
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
@@ -300,50 +304,56 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| entry.fi.data_dir)
|
||||
.flatten()
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
@@ -352,9 +362,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
@@ -363,29 +373,34 @@ pub(super) fn finish_active_multipart_part_transaction(
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
map.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
@@ -396,11 +411,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
@@ -1512,7 +1527,7 @@ impl LocalDisk {
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
if !parts.is_empty() {
|
||||
let anchor_path = directory.join(RETIREMENT_ANCHOR);
|
||||
let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
|
||||
@@ -1545,7 +1560,13 @@ impl LocalDisk {
|
||||
}
|
||||
// An anchor-less empty DD is only rmdir'ed. A new child makes
|
||||
// this fail closed; no recursive removal can consume its bytes.
|
||||
// A Windows pin refuses DELETE sharing, so the leaf pin is
|
||||
// released first; the object pin still holds the parent.
|
||||
#[cfg(windows)]
|
||||
drop(directory_pin);
|
||||
std::fs::remove_dir(&directory)?;
|
||||
#[cfg(not(windows))]
|
||||
drop(directory_pin);
|
||||
if sync {
|
||||
os::fsync_dir_std(&object_path)?;
|
||||
}
|
||||
@@ -1921,7 +1942,6 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
|
||||
+310
-851
File diff suppressed because it is too large
Load Diff
@@ -1899,6 +1899,10 @@ mod tests {
|
||||
&ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
mod_time: earlier(version.mod_time),
|
||||
// Without the object lock the rename waits for every
|
||||
// drive, so no early-ACK tail is still writing the copies
|
||||
// removed below.
|
||||
no_lock: true,
|
||||
..versioned
|
||||
},
|
||||
))
|
||||
|
||||
@@ -318,21 +318,6 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
|
||||
match rebalance_error_source(err) {
|
||||
Error::Lock(err) => is_rebalance_transient_lock_error(err),
|
||||
// No metadata was written when lock admission failed. Each attempt
|
||||
// must acquire its quorum again before loading or saving this run.
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket,
|
||||
object,
|
||||
..
|
||||
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return true;
|
||||
@@ -400,7 +385,9 @@ where
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(err) => {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1)
|
||||
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -732,82 +719,6 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
|
||||
let mut attempts = 0;
|
||||
retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(if attempts == 1 {
|
||||
resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)),
|
||||
"stop_rebalance",
|
||||
)
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("a later quorum-protected admission must be allowed to succeed");
|
||||
assert_eq!(attempts, 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
|
||||
for max_attempts in [0, 1, 3] {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, max_attempts, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)))
|
||||
})
|
||||
.await;
|
||||
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
|
||||
assert_eq!(attempts, max_attempts.max(1));
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&error),
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(if attempts == 1 {
|
||||
rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)
|
||||
} else {
|
||||
Error::OperationCanceled
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
|
||||
assert!(matches!(result, Err(Error::OperationCanceled)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
|
||||
for err in [
|
||||
@@ -820,20 +731,6 @@ mod error_source_tests {
|
||||
required: 3,
|
||||
achieved: 2,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: "data-bucket".to_string(),
|
||||
object: REBAL_META_NAME.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
|
||||
object: "other-metadata.bin".to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::other("stale rebalance run rejected: lock acquisition timed out"),
|
||||
Error::other("rebalance distributed run fence lost"),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
|
||||
@@ -4696,17 +4696,6 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
// Reusing a held writer additionally requires the same quorum, so repeated
|
||||
// clients must not give different weight to an otherwise identical domain.
|
||||
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
!left.is_empty()
|
||||
&& left.len() == right.len()
|
||||
&& left.iter().all(|client| {
|
||||
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
})
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -5019,11 +5008,6 @@ impl DiskHealthEntry {
|
||||
#[error("committed system metadata write requires repair by its owning writer")]
|
||||
pub(crate) struct SystemMetadataRepairRequired;
|
||||
|
||||
/// An inline metadata write was rejected before staging or commit began.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("system metadata write rejected before staging: {0}")]
|
||||
pub(crate) struct SystemMetadataWriteNotStarted(#[source] pub(crate) Error);
|
||||
|
||||
impl SetDisks {
|
||||
pub(in crate::set_disk) async fn persist_delete_marker_purge(
|
||||
&self,
|
||||
@@ -5661,9 +5645,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
// Distributed lock clients receive the raw ObjectKey; the pool
|
||||
// namespace is not part of the key stored by their backends.
|
||||
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -9898,7 +9882,7 @@ mod tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
@@ -9937,25 +9921,11 @@ mod tests {
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pools using identical backends lock the same raw object key"
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
|
||||
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
|
||||
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
|
||||
assert!(!same_distributed_lock_quorum(
|
||||
&[a.clone(), a.clone(), b.clone()],
|
||||
&[a.clone(), b.clone(), b]
|
||||
));
|
||||
assert!(!same_distributed_lock_quorum(&[], &[]));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
|
||||
@@ -375,10 +375,7 @@ impl SetDisks {
|
||||
return None;
|
||||
}
|
||||
|
||||
// A failed remote reconnect must retain its existing recovery monitor.
|
||||
if disk.is_local() {
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
|
||||
let endpoint = self.set_endpoints.get(i)?;
|
||||
@@ -500,26 +497,19 @@ impl SetDisks {
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
|
||||
if !new_disk.is_local() {
|
||||
let displaced = {
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
|
||||
_ => false,
|
||||
};
|
||||
// A GET probe or another reconnect may have filled this slot while
|
||||
// format I/O was pending. Only the successful publisher owns the
|
||||
// displaced monitor; a failed or superseded probe must retain it.
|
||||
if unchanged {
|
||||
new_disk.enable_health_check();
|
||||
slot.replace(new_disk)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
|
||||
_ => false,
|
||||
};
|
||||
if let Some(disk) = displaced {
|
||||
let _ = disk.close().await;
|
||||
// A GET probe or another reconnect may have filled this slot while
|
||||
// format I/O was pending. Do not replace its handle or start a
|
||||
// second health monitor for an unpublished handle.
|
||||
if unchanged {
|
||||
new_disk.enable_health_check();
|
||||
*slot = Some(new_disk);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ use super::object::{
|
||||
object_transaction_fencing_requested, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence,
|
||||
};
|
||||
use crate::api::config::storageclass;
|
||||
#[cfg(test)]
|
||||
use crate::bucket::metadata_sys::ObjectLockConfigState;
|
||||
use crate::bucket::quota::reservation;
|
||||
use crate::bucket::replication::ReplicationStatusType;
|
||||
@@ -4113,40 +4114,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !opts.versioned
|
||||
&& !crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
&& let Some(snapshot) = opts.object_lock_config_snapshot.as_deref()
|
||||
&& !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)
|
||||
{
|
||||
// Completion replaces the null version, not necessarily the latest
|
||||
// version inspected by the request-level Object Lock check.
|
||||
let current = self
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match current {
|
||||
Ok(existing) => {
|
||||
if check_object_lock_for_deletion_with_state(snapshot.state(), &existing, false)?.is_some() {
|
||||
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
}
|
||||
// A null delete marker has no protected payload to replace.
|
||||
Err(StorageError::MethodNotAllowed) => {}
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
if opts.data_movement
|
||||
&& opts.http_preconditions.is_some()
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||
@@ -10093,7 +10060,7 @@ mod tests {
|
||||
|
||||
// Real disk callbacks and health probes must share wall-clock deadlines.
|
||||
// Advancing Tokio time can expire unrelated healthy-drive probes.
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
|
||||
let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "mpu-quorum-success-physical";
|
||||
@@ -10120,8 +10087,6 @@ mod tests {
|
||||
local.observe_multipart_cleanup_for_test(&upload_path)
|
||||
}).collect::<Vec<_>>();
|
||||
let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
|
||||
// Fault only the held leg; healthy disks keep their normal I/O budgets.
|
||||
let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
|
||||
let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
|
||||
let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -10278,15 +10243,7 @@ mod tests {
|
||||
let size = copy_info.get_actual_size().expect("actual source size");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
|
||||
tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
|
||||
.await.expect("cross-key Copy cannot wait on the old successful logical owner")
|
||||
.unwrap_or_else(|error| {
|
||||
let health = disks.iter().enumerate().map(|(slot, disk)| {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
|
||||
let metrics = local.metrics_snapshot();
|
||||
(slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
|
||||
}).collect::<Vec<_>>();
|
||||
panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
|
||||
});
|
||||
.await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
|
||||
drop(copy_info);
|
||||
let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
|
||||
let mut actual = Vec::new();
|
||||
|
||||
@@ -4526,16 +4526,7 @@ impl SetDisks {
|
||||
"Set disk write quorum unavailable"
|
||||
);
|
||||
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
let error = to_object_err(write_err.into(), vec![bucket, object]);
|
||||
if direct_inline_commit
|
||||
&& is_meta_bucketname(bucket)
|
||||
&& opts.write_completion == crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair
|
||||
{
|
||||
// Inline writer setup only checks disk availability;
|
||||
// no shard or metadata write has been dispatched yet.
|
||||
return Err(Error::other(crate::set_disk::SystemMetadataWriteNotStarted(error)));
|
||||
}
|
||||
return Err(error);
|
||||
return Err(to_object_err(write_err.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
return Err(Error::other(format!("not enough disks to write: {errors:?}")));
|
||||
@@ -4967,20 +4958,7 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
// A write without a version id to a key that is not versioning-Enabled
|
||||
// replaces the key's null version in place, so the WORM gate must judge
|
||||
// that version too. The request-layer pre-check reads the latest
|
||||
// version, which is not the one replaced when a UUID version or a
|
||||
// delete marker sits above a retained null version.
|
||||
let worm_gate_version_id = opts.version_id.clone().or_else(|| {
|
||||
(!opts.versioned
|
||||
&& opts
|
||||
.object_lock_config_snapshot
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)))
|
||||
.then(|| Uuid::nil().to_string())
|
||||
});
|
||||
if let Some(version_id) = worm_gate_version_id.as_deref()
|
||||
if let Some(version_id) = opts.version_id.as_deref()
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
@@ -5000,10 +4978,9 @@ impl SetDisks {
|
||||
.await;
|
||||
match current {
|
||||
Ok(existing) => {
|
||||
let object_lock_config = opts
|
||||
.object_lock_config_snapshot
|
||||
.as_deref()
|
||||
.ok_or_else(|| Error::other("destructive PUT is missing its Object Lock configuration snapshot"))?;
|
||||
let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
|
||||
Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
|
||||
})?;
|
||||
// The WORM gate protects the locked version from local
|
||||
// overwrites; an authorized replication write passes it
|
||||
// only when the LWW merge below will judge every
|
||||
@@ -5034,8 +5011,6 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A null delete marker has no protected payload to replace.
|
||||
Err(StorageError::MethodNotAllowed) if version_id == Uuid::nil().to_string() => {}
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
@@ -9329,60 +9304,6 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// Judge the version a versionless destructive delete removes.
|
||||
///
|
||||
/// For a key that is not versioning-`Enabled`, a delete without a version
|
||||
/// id removes the `null` version, while the preceding Object Lock check
|
||||
/// judged the key's latest entry. When a UUID version or delete marker
|
||||
/// sits above a retained `null` version, the two differ and the retained
|
||||
/// version must be judged on its own. The caller holds the object's write
|
||||
/// lock. Plain unversioned keys and versioning-`Enabled` keys keep a single
|
||||
/// lookup: their latest entry is the version removed.
|
||||
///
|
||||
/// The lookup future is boxed from a synchronous wrapper so it leaves the
|
||||
/// delete futures' frames: `delete_object` already nests the boxed batch
|
||||
/// path, and inlining a second `get_object_info` overflowed the 4 MiB test
|
||||
/// stack on Linux.
|
||||
fn check_null_version_object_lock_delete<'a>(
|
||||
&'a self,
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
latest: &'a ObjectInfo,
|
||||
opts: &'a ObjectOptions,
|
||||
) -> impl Future<Output = Result<()>> + Send + 'a {
|
||||
Box::pin(self.check_null_version_object_lock_delete_inner(bucket, object, latest, opts))
|
||||
}
|
||||
|
||||
async fn check_null_version_object_lock_delete_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
latest: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
if is_meta_bucketname(bucket)
|
||||
|| opts.version_id.is_some()
|
||||
|| set_disk_delete_creates_delete_marker(opts)
|
||||
|| opts.transition.expire_restored
|
||||
|| latest.version_id.is_none_or(|version| version.is_nil())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let null_opts = ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
..opts.clone()
|
||||
};
|
||||
match self.get_object_info(bucket, object, &null_opts).await {
|
||||
Ok(existing) => check_object_lock_delete(&self.ctx, bucket, object, &existing, &null_opts).await,
|
||||
// A null delete marker has no protected payload to remove.
|
||||
Err(StorageError::MethodNotAllowed) => Ok(()),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read current metadata without cache admission or read-side repair. The
|
||||
/// outer batch owner already holds the sorted namespace write-lock set.
|
||||
pub(crate) async fn conditional_delete_current_snapshot(
|
||||
@@ -10101,17 +10022,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
del_errs[i] = Some(err);
|
||||
continue;
|
||||
}
|
||||
// Not gated on `source_missing`: with a delete marker on top the
|
||||
// latest-version lookup reports the key as missing while the null
|
||||
// version below it may still be retained.
|
||||
if object_lock_check_required
|
||||
&& let Err(err) = self
|
||||
.check_null_version_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts)
|
||||
.await
|
||||
{
|
||||
del_errs[i] = Some(err);
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.tier_delete_journal_api.is_some()
|
||||
&& !source_missing
|
||||
@@ -11184,8 +11094,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
opts.precondition_check(&goi)?;
|
||||
check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?;
|
||||
}
|
||||
self.check_null_version_object_lock_delete(bucket, object, &goi, &opts)
|
||||
.await?;
|
||||
let delete_marker_purge_candidate = opts
|
||||
.version_id
|
||||
.as_deref()
|
||||
|
||||
@@ -3692,177 +3692,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// The RPC lock service uses the raw ObjectKey and one backend per node,
|
||||
// even when the requesting sets carry different pool namespace strings.
|
||||
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
|
||||
use rustfs_lock::client::{LockClient, local::LocalClient};
|
||||
|
||||
let clients = || {
|
||||
(0..2)
|
||||
.map(|_| {
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let first = clients();
|
||||
let second = if shared { first.clone() } else { clients() };
|
||||
let mut pools = source.pools.clone();
|
||||
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
|
||||
for set in &mut Arc::make_mut(pool).disk_set {
|
||||
Arc::make_mut(set).lockers = lockers.clone();
|
||||
}
|
||||
}
|
||||
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
||||
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
|
||||
Arc::new(ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: Default::default(),
|
||||
pools,
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(state),
|
||||
ctx: Arc::clone(&source.ctx),
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
|
||||
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let first_lock = first
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("first pool lock");
|
||||
let second_lock = second
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("second pool lock");
|
||||
let guard = first_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"different pool namespaces still conflict on the same backend and object key"
|
||||
);
|
||||
drop(guard);
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(3))
|
||||
.await
|
||||
.expect("second writer should acquire after the first releases");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let missing = remove_pool_meta_shard(&store, 1).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cross-pool repair must not reenter the shared writer")
|
||||
.expect("pool metadata repair should return a result");
|
||||
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
assert!(
|
||||
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
|
||||
"the selected missing shard must be restored"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
|
||||
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let lock = target
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("target lock");
|
||||
let guard = lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("hold target lock");
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let heal_store = Arc::clone(&store);
|
||||
let mut heal = tokio::spawn(async move {
|
||||
heal_store
|
||||
.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
barrier.wait_until_external_heal_target_lock_attempted(),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must attempt the independent target lock");
|
||||
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
|
||||
drop(guard);
|
||||
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
|
||||
.await
|
||||
.expect("inspection should progress after target lock release")
|
||||
.expect("heal task should not panic")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
|
||||
|
||||
@@ -540,7 +540,6 @@ impl ECStore {
|
||||
// failures can still spawn recovery probes for peers that come up
|
||||
// after this node.
|
||||
let (mut disks, errs) = init_format::init_disks(
|
||||
&instance_ctx,
|
||||
&pool_eps.endpoints,
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
@@ -27353,282 +27352,4 @@ mod tests {
|
||||
.expect("recovery must preserve the local source object");
|
||||
assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn seed_retained_null_version(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
protection: &str,
|
||||
latest_is_marker: bool,
|
||||
) {
|
||||
let metadata = if protection == "legal-hold" {
|
||||
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
|
||||
} else {
|
||||
HashMap::from([
|
||||
("x-amz-object-lock-mode".to_string(), protection.to_string()),
|
||||
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
|
||||
])
|
||||
};
|
||||
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut original,
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed retained null version");
|
||||
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed newer unprotected UUID version");
|
||||
if latest_is_marker {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a UUID delete marker above the retained null version");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained null version must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read retained null version");
|
||||
assert_eq!(body, b"protected null version");
|
||||
}
|
||||
|
||||
/// A versionless delete removes the null version, so the Object Lock check
|
||||
/// must judge that version even when a UUID version or delete marker is
|
||||
/// the key's latest entry (rustfs/backlog#3017).
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_delete_checks_actual_target() {
|
||||
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let temp = tempfile::tempdir().expect("create null-version delete store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let configs = [
|
||||
VersioningConfiguration::default(),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
for (case, config) in configs.into_iter().enumerate() {
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
|
||||
"batch delete must judge the retained null version: {object}: {errors:?}"
|
||||
);
|
||||
let error = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("suspended single delete must also judge the retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
if protection == "GOVERNANCE" {
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
|
||||
bypass_governance: true,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
|
||||
assert!(
|
||||
store
|
||||
.get_object_reader(
|
||||
&bucket,
|
||||
&object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized deletion must make the retained null payload unreadable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A versionless PUT or multipart completion replaces the null version in
|
||||
/// place, so the commit-time WORM gate must judge that version even when a
|
||||
/// UUID version or delete marker is the key's latest entry
|
||||
/// (rustfs/backlog#3019).
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
|
||||
let temp = tempfile::tempdir().expect("create null-version overwrite store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
|
||||
let error = store
|
||||
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("PUT must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
|
||||
let upload = store
|
||||
.clone()
|
||||
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
|
||||
let part = store
|
||||
.clone()
|
||||
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart replacement");
|
||||
let error = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
|
||||
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
&object,
|
||||
&mut next_version,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a new UUID version must remain allowed");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,7 +318,10 @@ mod multipart_copy_heal_tests {
|
||||
let mut corrupt = original.clone();
|
||||
corrupt[0] ^= 0x80;
|
||||
std::fs::write(&shard, &corrupt).expect("actual bitrot corruption in shared physical shard");
|
||||
std::fs::File::open(&shard)
|
||||
// Windows only flushes a handle opened for writing.
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&shard)
|
||||
.expect("open corrupted actual shard")
|
||||
.sync_all()
|
||||
.expect("persist actual fault bytes");
|
||||
|
||||
@@ -44,32 +44,12 @@ const LEGACY_FORMAT_READ_CONCURRENCY: usize = 16;
|
||||
const LEGACY_FORMAT_BASE_MAX_BYTES: usize = 16 * 1024;
|
||||
const LEGACY_FORMAT_MAX_BYTES_PER_DISK: usize = 64;
|
||||
|
||||
pub async fn init_disks(
|
||||
instance_ctx: &InstanceContext,
|
||||
eps: &Endpoints,
|
||||
opt: &DiskOption,
|
||||
) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
|
||||
pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
|
||||
let mut futures = Vec::with_capacity(eps.as_ref().len());
|
||||
let local_disk_map = instance_ctx.local_disk_map();
|
||||
let local_disks = local_disk_map.read().await;
|
||||
|
||||
for ep in eps.as_ref().iter() {
|
||||
// Foundation disks may already serve RPC writes. Opening them again
|
||||
// with startup cleanup can move live staging out from under a writer.
|
||||
let registered = if ep.is_local {
|
||||
local_disks.get(&ep.to_string()).cloned().flatten()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
futures.push(async move {
|
||||
match registered {
|
||||
Some(disk) if disk.endpoint() == *ep => Ok(disk),
|
||||
Some(_) => Err(DiskError::InconsistentDisk),
|
||||
None => new_disk(ep, opt).await,
|
||||
}
|
||||
});
|
||||
futures.push(new_disk(ep, opt));
|
||||
}
|
||||
drop(local_disks);
|
||||
|
||||
let mut res = Vec::with_capacity(eps.as_ref().len());
|
||||
let mut errors = Vec::with_capacity(eps.as_ref().len());
|
||||
@@ -1050,7 +1030,6 @@ mod tests {
|
||||
}
|
||||
|
||||
let (disks, errors) = init_disks(
|
||||
&InstanceContext::new(),
|
||||
&Endpoints::from(endpoints),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
@@ -1070,91 +1049,6 @@ mod tests {
|
||||
(temp_dir, disks)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_disks_preserves_live_staging_on_registered_local_disks() {
|
||||
let (temp_dir, disks) = local_disks(1).await;
|
||||
let registered = disks[0].as_ref().expect("registered disk should exist");
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
record_local_disks(&instance_ctx, vec![registered.clone()]).await;
|
||||
let staging = temp_dir.path().join("disk-0/.rustfs.sys/tmp/in-flight/xl.meta");
|
||||
tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
|
||||
.await
|
||||
.expect("create staging directory");
|
||||
tokio::fs::write(&staging, b"in-flight pool metadata")
|
||||
.await
|
||||
.expect("stage metadata through the registered disk");
|
||||
|
||||
let (initialized, errors) = init_disks(
|
||||
&instance_ctx,
|
||||
&Endpoints::from(vec![registered.endpoint()]),
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
|
||||
assert_eq!(
|
||||
tokio::fs::read(&staging)
|
||||
.await
|
||||
.expect("startup must preserve live RPC staging"),
|
||||
b"in-flight pool metadata"
|
||||
);
|
||||
assert!(Arc::ptr_eq(registered, initialized[0].as_ref().expect("initialized disk should exist")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_disks_cleans_stale_staging_without_a_registered_disk() {
|
||||
let temp_dir = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint =
|
||||
Endpoint::try_from(temp_dir.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
|
||||
let staging = temp_dir.path().join(".rustfs.sys/tmp/stale/xl.meta");
|
||||
tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
|
||||
.await
|
||||
.expect("create stale staging directory");
|
||||
tokio::fs::write(&staging, b"stale pool metadata")
|
||||
.await
|
||||
.expect("write stale staging");
|
||||
|
||||
let (disks, errors) = init_disks(
|
||||
&InstanceContext::new(),
|
||||
&Endpoints::from(vec![endpoint]),
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
|
||||
assert!(disks[0].is_some());
|
||||
assert!(!staging.exists(), "a fresh disk open must still clear stale staging");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_disks_rejects_registered_disk_topology_mismatch() {
|
||||
let (_temp_dir, disks) = local_disks(1).await;
|
||||
let registered = disks[0].as_ref().expect("registered disk should exist");
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
record_local_disks(&instance_ctx, vec![registered.clone()]).await;
|
||||
let mut endpoint = registered.endpoint();
|
||||
endpoint.set_pool_index(1);
|
||||
|
||||
let (initialized, errors) = init_disks(
|
||||
&instance_ctx,
|
||||
&Endpoints::from(vec![endpoint]),
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(initialized[0].is_none(), "a registered disk must not be reopened in a different slot");
|
||||
assert_eq!(errors[0], Some(DiskError::InconsistentDisk));
|
||||
}
|
||||
|
||||
async fn format_bytes(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
|
||||
let mut snapshots = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
@@ -1864,7 +1758,6 @@ mod tests {
|
||||
disk.set_disk_id(None).await.expect("test disk ID should be reset");
|
||||
}
|
||||
let (prewarm_disks, prewarm_errors) = init_disks(
|
||||
&InstanceContext::new(),
|
||||
&prewarm_endpoints,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
|
||||
@@ -3295,7 +3295,6 @@ mod tests {
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
|
||||
let (mut disks, errs) = init_disks(
|
||||
&crate::runtime::global::current_ctx(),
|
||||
&endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
|
||||
@@ -1210,34 +1210,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
// The set-level commit judges the null version this completion
|
||||
// replaces; give it the same authoritative snapshot a PUT carries.
|
||||
let expected_incarnation_id = opts
|
||||
.expected_bucket_incarnation_id
|
||||
.ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
|
||||
let lifecycle_fence = opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
|
||||
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
|
||||
Some(snapshot) => Arc::clone(snapshot),
|
||||
None => {
|
||||
self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
|
||||
return Err(Error::other(
|
||||
"Object Lock snapshot does not hold valid target bucket generation and configuration fences",
|
||||
));
|
||||
}
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
opts.object_lock_config_snapshot = Some(snapshot);
|
||||
}
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
|
||||
@@ -925,13 +925,6 @@ impl MetaCacheEntriesSorted {
|
||||
|
||||
const METACACHE_STREAM_VERSION: u8 = 2;
|
||||
const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
|
||||
/// Upper bound on one incremental reservation while accumulating a single
|
||||
/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
|
||||
/// up to this bound so a legitimately large xl.meta does not pay one small
|
||||
/// read per chunk, while a corrupt length prefix still cannot force a
|
||||
/// reservation larger than the payload that has already arrived (see
|
||||
/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
|
||||
const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
|
||||
/// msgpack `false`: the entry-stream terminator, and the only byte a legacy
|
||||
/// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
|
||||
const METACACHE_TERMINATOR: u8 = 0xc2;
|
||||
@@ -1164,14 +1157,13 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
// A whole xl.meta can legitimately exceed the limit on its individual
|
||||
// msgpack values. Grow only as payload arrives, so a corrupt length
|
||||
// prefix cannot cause a speculative allocation of the advertised size.
|
||||
let mut chunk = METACACHE_READ_CHUNK_SIZE;
|
||||
while self.buf.len() < read_size {
|
||||
let start = self.buf.len();
|
||||
let extra = if lookahead {
|
||||
let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
|
||||
(read_size - start).clamp(min_chunk, chunk)
|
||||
(read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
|
||||
} else {
|
||||
(read_size - start).min(chunk)
|
||||
(read_size - start).min(METACACHE_READ_CHUNK_SIZE)
|
||||
};
|
||||
if let Err(e) = self.buf.try_reserve(extra) {
|
||||
let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
|
||||
@@ -1202,7 +1194,6 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
|
||||
}
|
||||
|
||||
self.pos = read_size;
|
||||
@@ -2164,76 +2155,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
|
||||
/// decode completely even when the aggregate stream exceeds the per-value
|
||||
/// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
|
||||
/// element.
|
||||
#[tokio::test]
|
||||
async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
|
||||
// ~2.9 KiB of metadata per entry matches the per-entry cost observed in
|
||||
// the #9120 production report; 6,000 entries push the stream past 16 MiB.
|
||||
const ENTRIES: usize = 6_000;
|
||||
let metadata = vec![0xab; 2_880];
|
||||
let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
|
||||
.map(|i| MetaCacheEntry {
|
||||
name: format!("bigdir/object-{i:05}"),
|
||||
metadata: metadata.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
writer.write(&entries).await.expect("wide listing must encode");
|
||||
writer.close().await.expect("stream must close");
|
||||
let bytes = output.into_inner();
|
||||
assert!(
|
||||
bytes.len() > 16 << 20,
|
||||
"fixture must exceed the per-value limit, got {} bytes",
|
||||
bytes.len()
|
||||
);
|
||||
let mut reader = MetacacheReader::new(Cursor::new(bytes));
|
||||
let decoded = reader.read_all().await.expect("wide listing must decode");
|
||||
assert_eq!(decoded, entries);
|
||||
}
|
||||
|
||||
/// The #9120 failure shape seen in production: one entry whose metadata is
|
||||
/// an xl.meta that grew past 16 MiB with its version history, listed among
|
||||
/// ordinary neighbors and skipped past without losing the framing.
|
||||
#[tokio::test]
|
||||
async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
|
||||
let small = |i: u8| MetaCacheEntry {
|
||||
name: format!("bigdir/object-{i}"),
|
||||
metadata: vec![i; 64],
|
||||
..Default::default()
|
||||
};
|
||||
let oversized = MetaCacheEntry {
|
||||
name: "bigdir/huge-versions".to_string(),
|
||||
metadata: vec![0xcd; (16 << 20) + 12_345],
|
||||
..Default::default()
|
||||
};
|
||||
let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
writer.write(&entries).await.expect("mixed listing must encode");
|
||||
writer.close().await.expect("stream must close");
|
||||
|
||||
let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
|
||||
assert_eq!(
|
||||
reader.peek().await.expect("first entry").expect("present"),
|
||||
entries[0],
|
||||
"an ordinary entry before the oversized one must decode"
|
||||
);
|
||||
reader.skip(1).await.expect("skip the first entry");
|
||||
assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
|
||||
assert_eq!(
|
||||
reader.peek().await.expect("oversized entry").expect("present"),
|
||||
oversized,
|
||||
"an entry past the per-value limit must round-trip whole"
|
||||
);
|
||||
reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
|
||||
assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
|
||||
let version_id = Uuid::new_v4();
|
||||
|
||||
@@ -128,20 +128,6 @@ impl Error {
|
||||
Error::TransientSkip { message: message.into() }
|
||||
}
|
||||
|
||||
/// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
|
||||
/// Callers must cut their own concurrency instead of retrying at full speed.
|
||||
pub(crate) fn is_slow_down(&self) -> bool {
|
||||
match self {
|
||||
Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
|
||||
Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
|
||||
Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
|
||||
message.contains("Please reduce your request rate")
|
||||
}
|
||||
Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a heal operation can be retried without changing its inputs.
|
||||
pub(crate) fn is_recoverable_heal(&self) -> bool {
|
||||
match classify_heal_error(self) {
|
||||
@@ -463,22 +449,6 @@ mod tests {
|
||||
use super::Error;
|
||||
use crate::heal::{DiskError, EcstoreError};
|
||||
|
||||
#[test]
|
||||
fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
|
||||
assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
|
||||
assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
|
||||
assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
|
||||
assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
|
||||
assert!(
|
||||
Error::TaskExecutionFailed {
|
||||
message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
|
||||
}
|
||||
.is_slow_down()
|
||||
);
|
||||
assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
|
||||
assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_metadata_failures_keep_their_typed_recovery_classification() {
|
||||
use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::error::{HealErrorClass, classify_heal_error};
|
||||
use crate::heal::{
|
||||
pacing::HealPageRate,
|
||||
progress::{HealProgress, add_bytes, increment_counter},
|
||||
resume::{
|
||||
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
|
||||
@@ -129,12 +128,12 @@ pub struct ErasureSetHealer {
|
||||
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
||||
replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
|
||||
work_parent: Option<super::manager::HealWork>,
|
||||
mainline_pacer: Option<super::pacing::ForegroundPacing>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
}
|
||||
|
||||
async fn acquire_page_permit(
|
||||
semaphore: Arc<Semaphore>,
|
||||
rate: &HealPageRate,
|
||||
pacer: Option<&super::pacing::MainlinePacer>,
|
||||
cancel: &tokio_util::sync::CancellationToken,
|
||||
) -> Result<tokio::sync::OwnedSemaphorePermit> {
|
||||
let acquire = || async {
|
||||
@@ -147,16 +146,18 @@ async fn acquire_page_permit(
|
||||
let mut paid_pause = false;
|
||||
loop {
|
||||
let permit = acquire().await?;
|
||||
// Keep the real permit on the low-pressure path. Every acquisition
|
||||
// gets a fresh decision, including a waiter that queued a second
|
||||
// time. One completed pause is a bounded minimum-progress grant.
|
||||
match rate.admission_decision() {
|
||||
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
|
||||
drop(permit);
|
||||
paid_pause = rate.wait_after_admission(cancel, pressure).await?;
|
||||
continue;
|
||||
if let Some(pacer) = pacer {
|
||||
// Keep the real permit on the low-pressure path. Every acquisition
|
||||
// gets a fresh decision, including a waiter that queued a second
|
||||
// time. One completed pause is a bounded minimum-progress grant.
|
||||
match pacer.admission_decision() {
|
||||
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
|
||||
drop(permit);
|
||||
paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
return Ok(permit);
|
||||
}
|
||||
@@ -165,7 +166,7 @@ async fn acquire_page_permit(
|
||||
#[cfg(test)]
|
||||
mod mainline_pacing_tests {
|
||||
use super::*;
|
||||
use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
|
||||
use crate::heal::pacing::{MainlinePacer, TestPressure};
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -179,15 +180,15 @@ mod mainline_pacing_tests {
|
||||
.await
|
||||
.expect("existing object owns capacity");
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
|
||||
let pacer =
|
||||
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
|
||||
let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
|
||||
let pacer = Arc::new(
|
||||
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
|
||||
);
|
||||
let cancel = CancellationToken::new();
|
||||
let waiting = tokio::spawn({
|
||||
let semaphore = semaphore.clone();
|
||||
let rate = rate.clone();
|
||||
let pacer = pacer.clone();
|
||||
let cancel = cancel.clone();
|
||||
async move { acquire_page_permit(semaphore, &rate, &cancel).await }
|
||||
async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
@@ -199,13 +200,12 @@ mod mainline_pacing_tests {
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
let deadline = tokio::time::timeout(
|
||||
Duration::from_millis(10),
|
||||
acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
|
||||
acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(deadline.is_err());
|
||||
assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
|
||||
let unpaced = HealPageRate::new(1, None);
|
||||
let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
|
||||
let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
|
||||
.await
|
||||
.expect("unpaced admission");
|
||||
assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
|
||||
@@ -221,11 +221,10 @@ mod mainline_pacing_tests {
|
||||
let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
|
||||
let pause = Duration::from_millis(250);
|
||||
let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
|
||||
let rate = HealPageRate::new(1, Some(pacer));
|
||||
let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
|
||||
let cancel = CancellationToken::new();
|
||||
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
|
||||
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
|
||||
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
|
||||
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(second.as_mut()).is_pending());
|
||||
drop(occupied);
|
||||
@@ -330,20 +329,6 @@ impl ErasureSetHealer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Page fan-out for one whole erasure-set pass (stable across buckets).
|
||||
fn pass_page_concurrency_limit(&self) -> usize {
|
||||
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
|
||||
// Versions of the same object must not race without its namespace lock.
|
||||
1
|
||||
} else {
|
||||
Self::effective_heal_page_object_concurrency_for_source(
|
||||
self.source,
|
||||
self.heal_opts.scan_mode,
|
||||
self.replacement_task_id.is_some(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Map the shared heal error class onto this healer's per-object actions.
|
||||
///
|
||||
/// `classify_heal_error` is the only classifier; this adapter decides what
|
||||
@@ -388,7 +373,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
|
||||
self.mainline_pacer = pacer;
|
||||
self
|
||||
}
|
||||
@@ -479,7 +464,12 @@ impl ErasureSetHealer {
|
||||
None
|
||||
};
|
||||
let buckets = admin_buckets.as_deref().unwrap_or(buckets);
|
||||
if checkpoint_manager.admin_completed().await {
|
||||
if checkpoint_manager
|
||||
.get_checkpoint()
|
||||
.await
|
||||
.admin
|
||||
.is_some_and(|admin| admin.completed)
|
||||
{
|
||||
self.initialize_progress(buckets, &resume_manager.get_state().await).await;
|
||||
return Ok(());
|
||||
}
|
||||
@@ -500,7 +490,14 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
if self.admin_task.is_some() {
|
||||
checkpoint_manager.complete_admin().await?;
|
||||
let mut admin = checkpoint_manager
|
||||
.get_checkpoint()
|
||||
.await
|
||||
.admin
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
|
||||
admin.completed = true;
|
||||
admin.outcome.finish(None);
|
||||
checkpoint_manager.set_admin(admin).await?;
|
||||
} else {
|
||||
checkpoint_manager.cleanup().await?;
|
||||
resume_manager.cleanup().await?;
|
||||
@@ -721,10 +718,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
return Err(Error::DanglingDeleteDeferred { retry_not_before });
|
||||
}
|
||||
// One rate controller for the whole pass: SlowDown cool-down and page
|
||||
// fan-out persist across buckets instead of resetting at each one.
|
||||
let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
|
||||
let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -732,17 +726,14 @@ impl ErasureSetHealer {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
current_bucket_index,
|
||||
current_object_index,
|
||||
current_bucket_index = checkpoint.current_bucket_index,
|
||||
current_object_index = checkpoint.current_object_index,
|
||||
state = "resuming",
|
||||
"Erasure set resumed"
|
||||
);
|
||||
|
||||
// 2. initialize progress
|
||||
self.initialize_progress(buckets, &state).await;
|
||||
// Snapshot before any baseline write: set_progress_baseline overwrites
|
||||
// the saved generation, which would hide a real generation mismatch.
|
||||
let checkpoint = checkpoint_manager.progress_snapshot().await;
|
||||
let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
|
||||
let baseline = self.progress.read().await;
|
||||
(
|
||||
@@ -769,7 +760,17 @@ impl ErasureSetHealer {
|
||||
let mut successful_objects = state.successful_objects;
|
||||
let mut failed_objects = state.failed_objects;
|
||||
let mut skipped_objects = state.skipped_objects;
|
||||
let checkpoint_has_progress = checkpoint.has_progress();
|
||||
let checkpoint_has_progress = checkpoint.baseline_known
|
||||
|| checkpoint.successful_objects > 0
|
||||
|| checkpoint.failed_object_count > 0
|
||||
|| checkpoint.skipped_object_count > 0
|
||||
|| checkpoint.skipped_new_versions > 0
|
||||
|| checkpoint.skipped_ilm_expired > 0
|
||||
|| checkpoint.processed_bytes > 0
|
||||
|| checkpoint.total_objects > 0
|
||||
|| checkpoint.total_bytes > 0
|
||||
|| checkpoint.baseline_generation.is_some()
|
||||
|| checkpoint.counter_unknown;
|
||||
let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
|
||||
let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
|
||||
if checkpoint_has_progress {
|
||||
@@ -853,7 +854,6 @@ impl ErasureSetHealer {
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
state.start_time,
|
||||
&page_rate,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1020,14 +1020,14 @@ impl ErasureSetHealer {
|
||||
if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
|
||||
if self.replacement_task_id.is_some() {
|
||||
let state = resume_manager.get_state().await;
|
||||
let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
if state.can_retry()
|
||||
&& failed_objects == 0
|
||||
&& failed_buckets == 0
|
||||
&& skipped_objects > 0
|
||||
&& skipped_objects == grace_objects
|
||||
&& !counter_unknown
|
||||
&& let Some(retry_not_before) = retry_not_before
|
||||
&& skipped_objects == checkpoint.dangling_delete_grace_objects
|
||||
&& !checkpoint.counter_unknown
|
||||
&& let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
|
||||
{
|
||||
// Publish the generation's rewind and wait first. If the
|
||||
// checkpoint reset is interrupted, initialization reapplies
|
||||
@@ -1167,9 +1167,14 @@ impl ErasureSetHealer {
|
||||
|
||||
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
|
||||
let checkpoint_key = compose_key(&object_key, None);
|
||||
if checkpoint_manager.is_object_settled(&checkpoint_key).await {
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
if checkpoint.processed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.failed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.skipped_objects.contains(&checkpoint_key)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
drop(checkpoint);
|
||||
|
||||
self.verify_replacement_identity_fence("pool metadata").await?;
|
||||
resume_manager
|
||||
@@ -1423,7 +1428,7 @@ impl ErasureSetHealer {
|
||||
|
||||
/// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
async fn heal_bucket_with_resume(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1437,7 +1442,6 @@ impl ErasureSetHealer {
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
started_at_secs: u64,
|
||||
page_rate: &Arc<HealPageRate>,
|
||||
) -> Result<bool> {
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -1487,6 +1491,17 @@ impl ErasureSetHealer {
|
||||
// detects a non-advancing cursor even though next_marker embeds a
|
||||
// transient cache id that changes between identical pages.
|
||||
let mut previous_page_last: Option<(String, Option<String>)> = None;
|
||||
let page_concurrency_limit =
|
||||
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
|
||||
// Versions of the same object must not race without its namespace lock.
|
||||
1
|
||||
} else {
|
||||
Self::effective_heal_page_object_concurrency_for_source(
|
||||
self.source,
|
||||
self.heal_opts.scan_mode,
|
||||
self.replacement_task_id.is_some(),
|
||||
)
|
||||
};
|
||||
let in_flight = Arc::new(AtomicUsize::new(0));
|
||||
// Per-bucket sample caps for per-object warn! lines: a flapping rebuild
|
||||
// disk can fail/skip hundreds of thousands of versions in one sweep, so
|
||||
@@ -1508,7 +1523,9 @@ impl ErasureSetHealer {
|
||||
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
|
||||
|
||||
loop {
|
||||
page_rate.wait_before_page(&self.cancel_token).await?;
|
||||
if let Some(pacer) = &self.mainline_pacer {
|
||||
pacer.wait(&self.cancel_token).await?;
|
||||
}
|
||||
self.verify_replacement_identity_fence("page scan").await?;
|
||||
if let Some(expected) = incarnation {
|
||||
self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
|
||||
@@ -1530,8 +1547,9 @@ impl ErasureSetHealer {
|
||||
.await?
|
||||
};
|
||||
let page_is_empty = objects.is_empty();
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
let page_resume_index = *current_object_index;
|
||||
let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
|
||||
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
|
||||
let mut page_tasks = FuturesUnordered::new();
|
||||
let mut completed_in_page = 0usize;
|
||||
|
||||
@@ -1545,7 +1563,10 @@ impl ErasureSetHealer {
|
||||
|
||||
// Per-version dedup identity — the single canonical key.
|
||||
let key = compose_key(&item.name, item.version_id.as_deref());
|
||||
if checkpoint_manager.is_object_settled(&key).await {
|
||||
if checkpoint.processed_objects.contains(&key)
|
||||
|| checkpoint.failed_objects.contains(&key)
|
||||
|| checkpoint.skipped_objects.contains(&key)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1703,7 +1724,7 @@ impl ErasureSetHealer {
|
||||
let target_endpoints = self.target_endpoints.clone();
|
||||
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
|
||||
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
|
||||
let page_rate = page_rate.clone();
|
||||
let mainline_pacer = self.mainline_pacer.clone();
|
||||
|
||||
let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
|
||||
let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
|
||||
@@ -1715,15 +1736,14 @@ impl ErasureSetHealer {
|
||||
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
|
||||
let work = async move {
|
||||
let mut retries = 0;
|
||||
let mut saw_slow_down = false;
|
||||
let mut admin_identity = admin_identity;
|
||||
let mut admin_task_id = admin_task_id;
|
||||
loop {
|
||||
let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
|
||||
let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
|
||||
|
||||
let permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
|
||||
};
|
||||
|
||||
let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
|
||||
@@ -1731,8 +1751,7 @@ impl ErasureSetHealer {
|
||||
if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
|
||||
let (result, canonical) =
|
||||
admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
|
||||
let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
|
||||
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
|
||||
return (dedup_key, object_name, version_id, result, canonical);
|
||||
}
|
||||
|
||||
// Always go through heal_object. Genuine absence flows through
|
||||
@@ -1834,23 +1853,18 @@ impl ErasureSetHealer {
|
||||
|
||||
drop(in_flight_guard);
|
||||
drop(permit);
|
||||
if matches!(&result.1, Err(err) if err.is_slow_down()) {
|
||||
saw_slow_down = true;
|
||||
}
|
||||
if retry_replacement
|
||||
&& retries < REPLACEMENT_OBJECT_MAX_RETRIES
|
||||
&& matches!(&result.1, Err(Error::TransientSkip { .. }))
|
||||
{
|
||||
retries += 1;
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => {
|
||||
return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
|
||||
}
|
||||
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
|
||||
_ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
|
||||
return (dedup_key, object_name, version_id, result, None);
|
||||
}
|
||||
};
|
||||
page_tasks.push(async move {
|
||||
@@ -1861,7 +1875,7 @@ impl ErasureSetHealer {
|
||||
};
|
||||
match result {
|
||||
Ok(result) => result,
|
||||
Err(error) => (key, object, version, (0, Err(error), false), None),
|
||||
Err(error) => (key, object, version, (0, Err(error)), None),
|
||||
}
|
||||
} else {
|
||||
work.await
|
||||
@@ -1870,16 +1884,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
|
||||
let (object_size, result, saw_slow_down) = result;
|
||||
// Exactly one rate observation per finished object, including
|
||||
// SlowDown that later succeeded on a replacement retry.
|
||||
page_rate.observe_object_outcome(
|
||||
&match &result {
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
saw_slow_down,
|
||||
);
|
||||
let (object_size, result) = result;
|
||||
let mut telemetry_unknown = false;
|
||||
let checkpoint_outcome = match result {
|
||||
Ok(true) => {
|
||||
@@ -2477,7 +2482,6 @@ mod resume_loop_tests {
|
||||
use super::{
|
||||
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
|
||||
};
|
||||
use crate::heal::pacing::HealPageRate;
|
||||
use crate::heal::progress::HealProgress;
|
||||
use crate::heal::resume::{
|
||||
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
|
||||
@@ -3059,7 +3063,6 @@ mod resume_loop_tests {
|
||||
let mut successful = 0u64;
|
||||
let mut failed = 0u64;
|
||||
let mut skipped = 0u64;
|
||||
let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
|
||||
let result = env
|
||||
.healer
|
||||
.heal_bucket_with_resume(
|
||||
@@ -3074,7 +3077,6 @@ mod resume_loop_tests {
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
state.start_time,
|
||||
&page_rate,
|
||||
)
|
||||
.await;
|
||||
(processed, successful, failed, skipped, result.map(|_| ()))
|
||||
@@ -3142,7 +3144,6 @@ mod resume_loop_tests {
|
||||
let mut skipped = 0;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
|
||||
let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
|
||||
let error = healer
|
||||
.heal_bucket_with_resume(
|
||||
"b",
|
||||
@@ -3156,7 +3157,6 @@ mod resume_loop_tests {
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
started_at,
|
||||
&page_rate,
|
||||
)
|
||||
.await
|
||||
.expect_err("a remounted target must not begin a new page scan");
|
||||
|
||||
@@ -810,14 +810,7 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
|
||||
fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
|
||||
active_heals
|
||||
.iter()
|
||||
.find(|(_, task)| {
|
||||
PriorityHealQueue::make_dedup_key_for_owner(
|
||||
&task.heal_type,
|
||||
&task.options,
|
||||
task.source,
|
||||
task.expected_mrf_bucket_incarnation_id,
|
||||
) == key
|
||||
})
|
||||
.find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
|
||||
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
||||
}
|
||||
|
||||
|
||||
@@ -611,104 +611,7 @@ impl PriorityHealQueue {
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
|
||||
Self::make_dedup_key_for_owner(
|
||||
&request.heal_type,
|
||||
&request.options,
|
||||
request.source,
|
||||
request.expected_mrf_bucket_incarnation_id,
|
||||
)
|
||||
}
|
||||
|
||||
/// Bound MRF work can only merge with work for the same bucket lifetime.
|
||||
/// Length-prefix user-controlled components so a key cannot forge another
|
||||
/// version, scope, or incarnation through delimiter characters.
|
||||
pub(super) fn make_dedup_key_for_owner(
|
||||
heal_type: &HealType,
|
||||
options: &HealOptions,
|
||||
source: HealRequestSource,
|
||||
incarnation: Option<Uuid>,
|
||||
) -> String {
|
||||
if source == HealRequestSource::Mrf
|
||||
&& let Some(incarnation) = incarnation
|
||||
{
|
||||
use std::fmt::Write as _;
|
||||
|
||||
let (pool, set) = heal_scope_indices(heal_type, options);
|
||||
let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
|
||||
let mut part = |value: &str| {
|
||||
// Formatting into a String is infallible.
|
||||
let _ = write!(key, "{}:{value}", value.len());
|
||||
};
|
||||
match heal_type {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
part("object");
|
||||
part(bucket);
|
||||
part(object);
|
||||
match version_id {
|
||||
None => part("none"),
|
||||
Some(version_id) => {
|
||||
part("some");
|
||||
part(version_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
part("metadata");
|
||||
part(bucket);
|
||||
part(object);
|
||||
}
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
part("ecdecode");
|
||||
part(bucket);
|
||||
part(object);
|
||||
match version_id {
|
||||
None => part("none"),
|
||||
Some(version_id) => {
|
||||
part("some");
|
||||
part(version_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealType::DeleteMarkerPurge {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
purge,
|
||||
} => {
|
||||
part("delete-marker-purge");
|
||||
part(bucket);
|
||||
part(object);
|
||||
part(version_id);
|
||||
part(&purge.bucket_incarnation_id.to_string());
|
||||
part(&purge.marker_incarnation_id.to_string());
|
||||
part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
part("bucket");
|
||||
part(bucket);
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
part("prefix");
|
||||
part(bucket);
|
||||
part(prefix);
|
||||
}
|
||||
// Cluster and erasure-set work are not bucket-bound; preserve
|
||||
// their existing keys, including contains_erasure_set lookups.
|
||||
HealType::Cluster | HealType::ErasureSet { .. } => {
|
||||
return Self::make_dedup_key_for_scope(heal_type, options);
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
Self::make_dedup_key_for_scope(heal_type, options)
|
||||
Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
|
||||
}
|
||||
|
||||
pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {
|
||||
|
||||
@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
|
||||
request: &HealRequest,
|
||||
config: &HealConfig,
|
||||
provider: &Option<WorkloadSnapshotProviderRef>,
|
||||
) -> Option<crate::heal::pacing::ForegroundPacing> {
|
||||
) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
|
||||
let pace_running = request.source == HealRequestSource::Admin
|
||||
|| (request.source == HealRequestSource::AutoHeal
|
||||
&& matches!(&request.heal_type, HealType::ErasureSet { .. })
|
||||
@@ -43,13 +43,14 @@ pub(super) fn running_mainline_pacer(
|
||||
return None;
|
||||
}
|
||||
provider.as_ref().and_then(|provider| {
|
||||
crate::heal::pacing::ForegroundPacing::new(
|
||||
crate::heal::pacing::MainlinePacer::new(
|
||||
provider.clone(),
|
||||
config.mainline_read_utilization_high_percent,
|
||||
config.mainline_write_utilization_high_percent,
|
||||
config.mainline_max_sleep,
|
||||
request.source,
|
||||
)
|
||||
.map(Arc::new)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -117,453 +117,6 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
|
||||
let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(incarnation);
|
||||
request.options.pool_index = Some(0);
|
||||
request.options.set_index = Some(0);
|
||||
request
|
||||
}
|
||||
|
||||
async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
|
||||
let old_incarnation = Uuid::new_v4();
|
||||
let new_incarnation = Uuid::new_v4();
|
||||
let old_request = incarnation_bound_mrf_fallback(old_incarnation);
|
||||
let old_id = old_request.id.clone();
|
||||
match existing_state {
|
||||
"queued" => {
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(old_request)
|
||||
.await
|
||||
.expect("old fallback must be admitted");
|
||||
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
|
||||
}
|
||||
"active" => {
|
||||
manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
|
||||
}
|
||||
"retrying" => {
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
old_id.clone(),
|
||||
RetryingHeal {
|
||||
request: old_request,
|
||||
error: "retry pending".to_owned(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
other => panic!("unknown fixture state: {other}"),
|
||||
}
|
||||
|
||||
let new_request = incarnation_bound_mrf_fallback(new_incarnation);
|
||||
let new_id = new_request.id.clone();
|
||||
assert_ne!(new_id, old_id, "the fallback must use a new request ID");
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(new_request)
|
||||
.await
|
||||
.expect("new incarnation admission must be decided");
|
||||
assert_eq!(
|
||||
receipt.result,
|
||||
HealAdmissionResult::Accepted,
|
||||
"a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
|
||||
);
|
||||
assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
|
||||
let queue = manager.heal_queue.lock().await;
|
||||
assert!(
|
||||
queue
|
||||
.requests()
|
||||
.any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
|
||||
);
|
||||
if existing_state == "queued" {
|
||||
assert!(
|
||||
queue
|
||||
.requests()
|
||||
.any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("active").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
|
||||
}
|
||||
|
||||
fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
|
||||
*id = version_id.map(str::to_owned);
|
||||
} else {
|
||||
unreachable!("fallback fixture must be an object request");
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
|
||||
for version in [Uuid::nil(), Uuid::new_v4()] {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let version_id = version.to_string();
|
||||
let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
|
||||
let original_id = original.id.clone();
|
||||
let accepted = manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first queued fallback");
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
let merged = manager
|
||||
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
|
||||
.await
|
||||
.expect("duplicate queued fallback");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, original_id);
|
||||
assert_eq!(manager.heal_queue.lock().await.len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let original = incarnation_bound_mrf_fallback(incarnation);
|
||||
let original_id = original.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let same = manager
|
||||
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
|
||||
.await
|
||||
.expect("same-incarnation fallback");
|
||||
assert_eq!(same.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(same.task_id, original_id);
|
||||
|
||||
let mut versioned = incarnation_bound_mrf_fallback(incarnation);
|
||||
versioned.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
};
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(versioned)
|
||||
.await
|
||||
.expect("different version")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
|
||||
other_scope.options.pool_index = Some(1);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(other_scope)
|
||||
.await
|
||||
.expect("different pool")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
same_id_other_incarnation.id = original_id;
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(same_id_other_incarnation)
|
||||
.await
|
||||
.expect("same-ID conflict must be decided")
|
||||
.result,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let bound = incarnation_bound_mrf_fallback(incarnation);
|
||||
let mut unbound = bound.clone();
|
||||
unbound.expected_mrf_bucket_incarnation_id = None;
|
||||
let mut non_mrf = bound.clone();
|
||||
non_mrf.source = HealRequestSource::ReadRepair;
|
||||
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key(&non_mrf),
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
|
||||
"an unrelated source retains its previous scope key even if the MRF-only field is set"
|
||||
);
|
||||
assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
|
||||
|
||||
let mut left = bound.clone();
|
||||
left.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "part:a".to_owned(),
|
||||
version_id: Some("b".to_owned()),
|
||||
};
|
||||
let mut right = bound.clone();
|
||||
right.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "part".to_owned(),
|
||||
version_id: Some("a:b".to_owned()),
|
||||
};
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
|
||||
"the legacy delimiter-only key demonstrates the collision"
|
||||
);
|
||||
assert_ne!(
|
||||
PriorityHealQueue::make_dedup_key(&left),
|
||||
PriorityHealQueue::make_dedup_key(&right),
|
||||
"length-prefixed object and version components must not alias"
|
||||
);
|
||||
let mut forged = unbound;
|
||||
forged.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: PriorityHealQueue::make_dedup_key(&bound),
|
||||
version_id: None,
|
||||
};
|
||||
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
for decode in [false, true] {
|
||||
let make_request = |object: &str, version_id: Option<&str>| {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
let bucket = "mrf-recreated-bucket".to_owned();
|
||||
let object = object.to_owned();
|
||||
let version_id = version_id.map(str::to_owned);
|
||||
request.heal_type = if decode {
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
} else {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
};
|
||||
request
|
||||
};
|
||||
let absent = make_request("object", None);
|
||||
let empty = make_request("object", Some(""));
|
||||
let forged_object = make_request("object4:some0:", None);
|
||||
let forged_version = make_request("object", Some("4:none"));
|
||||
let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
assert!(keys.iter().skip(index + 1).all(|other| key != other));
|
||||
}
|
||||
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
|
||||
assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
|
||||
assert_eq!(queue.len(), 2);
|
||||
|
||||
for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
|
||||
let mut legacy_absent = absent.clone();
|
||||
let mut legacy_empty = empty.clone();
|
||||
legacy_absent.source = source;
|
||||
legacy_empty.source = source;
|
||||
if source == HealRequestSource::Mrf {
|
||||
legacy_absent.expected_mrf_bucket_incarnation_id = None;
|
||||
legacy_empty.expected_mrf_bucket_incarnation_id = None;
|
||||
}
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key(&legacy_absent),
|
||||
PriorityHealQueue::make_dedup_key(&legacy_empty),
|
||||
"unbound and non-MRF work retain the legacy empty-version key"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
let old_id = old.id.clone();
|
||||
let old_key = PriorityHealQueue::make_dedup_key(&old);
|
||||
let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
let new_id = new.id.clone();
|
||||
let new_key = PriorityHealQueue::make_dedup_key(&new);
|
||||
let mut forced_old = old.clone();
|
||||
forced_old.id = Uuid::new_v4().to_string();
|
||||
forced_old.force_start = true;
|
||||
let forced_id = forced_old.id.clone();
|
||||
|
||||
assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
|
||||
queue
|
||||
.remove_request_id(&old_id)
|
||||
.expect("remove the old incarnation representative");
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
|
||||
assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
|
||||
assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
|
||||
assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
|
||||
assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
|
||||
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
|
||||
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(old)
|
||||
.await
|
||||
.expect("old fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let request = incarnation_bound_mrf_fallback(incarnation);
|
||||
let request_id = request.id.clone();
|
||||
let mut intent = MrfIntent {
|
||||
bucket: Arc::from("mrf-recreated-bucket"),
|
||||
object: Arc::from("object"),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
};
|
||||
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
|
||||
let target = MrfRepairNoticeTarget {
|
||||
bucket: Arc::clone(&intent.bucket),
|
||||
object: Arc::clone(&intent.object),
|
||||
version_id: intent.version_id,
|
||||
kind: intent.kind,
|
||||
scope: intent.scope,
|
||||
delete_marker_purge: None,
|
||||
lease: intent.lease,
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
|
||||
.await
|
||||
.expect("durable admission must be decided");
|
||||
assert_eq!(
|
||||
receipt.result,
|
||||
HealAdmissionResult::Accepted,
|
||||
"a durable new incarnation needs its own owner"
|
||||
);
|
||||
assert_eq!(receipt.task_id, request_id);
|
||||
assert!(
|
||||
manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.requests()
|
||||
.any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.try_in_flight_durable_mrf_anchors()
|
||||
.expect("stable owner snapshot")
|
||||
.contains(&anchor)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
|
||||
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
|
||||
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let object = format!("same-generation-{}", Uuid::new_v4());
|
||||
let make_request = || {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
request.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: object.clone(),
|
||||
version_id: None,
|
||||
};
|
||||
request
|
||||
};
|
||||
let old = make_request();
|
||||
let old_id = old.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(old)
|
||||
.await
|
||||
.expect("old fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut intent = MrfIntent {
|
||||
bucket: Arc::from("mrf-recreated-bucket"),
|
||||
object: Arc::from(object.as_str()),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
};
|
||||
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
|
||||
let target = MrfRepairNoticeTarget {
|
||||
bucket: Arc::clone(&intent.bucket),
|
||||
object: Arc::clone(&intent.object),
|
||||
version_id: intent.version_id,
|
||||
kind: intent.kind,
|
||||
scope: intent.scope,
|
||||
delete_marker_purge: None,
|
||||
lease: intent.lease,
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let merged = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
|
||||
.await
|
||||
.expect("durable same-incarnation admission");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, old_id);
|
||||
assert!(
|
||||
manager
|
||||
.try_in_flight_durable_mrf_anchors()
|
||||
.expect("stable owner snapshot")
|
||||
.contains(&anchor)
|
||||
);
|
||||
}
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
task_id: "fixture-token".to_string(),
|
||||
|
||||
+13
-232
@@ -7,14 +7,8 @@ use rustfs_concurrency::{
|
||||
workload::{ForegroundPressure, foreground_pressure},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU32, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{sync::Mutex as AsyncMutex, time::Instant};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -28,19 +22,17 @@ pub(crate) enum PacingDecision {
|
||||
Wait(Option<ForegroundPressure>),
|
||||
}
|
||||
|
||||
/// Foreground watermarks for one heal execution. Constructed once per request;
|
||||
/// disabled when watermarks are zero or no workload provider is configured.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ForegroundPacing {
|
||||
/// Cooperative pacing for one heal execution, not a storage admission permit.
|
||||
pub(crate) struct MainlinePacer {
|
||||
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
|
||||
read_high: usize,
|
||||
write_high: usize,
|
||||
pause: Duration,
|
||||
source: HealRequestSource,
|
||||
state: Arc<AsyncMutex<PacingState>>,
|
||||
state: Mutex<PacingState>,
|
||||
}
|
||||
|
||||
impl ForegroundPacing {
|
||||
impl MainlinePacer {
|
||||
pub(crate) fn new(
|
||||
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
|
||||
read_high: usize,
|
||||
@@ -57,7 +49,7 @@ impl ForegroundPacing {
|
||||
write_high: write_high.min(100),
|
||||
pause: pause.min(Duration::from_secs(1)),
|
||||
source,
|
||||
state: Arc::new(AsyncMutex::new(PacingState::default())),
|
||||
state: Mutex::new(PacingState::default()),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -133,142 +125,6 @@ impl ForegroundPacing {
|
||||
}
|
||||
}
|
||||
|
||||
/// Single owner of "how hard may this heal pass push storage?".
|
||||
///
|
||||
/// One instance spans the whole erasure-set scan (all buckets and pages):
|
||||
/// - foreground admission pressure (`ForegroundPacing`)
|
||||
/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
|
||||
///
|
||||
/// Call `observe_object_outcome` exactly once per finished object so a single
|
||||
/// SlowDown cannot double-reduce concurrency.
|
||||
pub(crate) struct HealPageRate {
|
||||
base_concurrency: usize,
|
||||
/// 0 = full base concurrency; each SlowDown increases the reduction.
|
||||
slow_down_step: AtomicU32,
|
||||
slow_down_until: std::sync::Mutex<Option<Instant>>,
|
||||
foreground: Option<ForegroundPacing>,
|
||||
}
|
||||
|
||||
impl HealPageRate {
|
||||
/// Always-on controller. Foreground yield is optional; SlowDown control is not.
|
||||
pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
|
||||
Self {
|
||||
base_concurrency: base_concurrency.max(1),
|
||||
slow_down_step: AtomicU32::new(0),
|
||||
slow_down_until: std::sync::Mutex::new(None),
|
||||
foreground,
|
||||
}
|
||||
}
|
||||
|
||||
/// Concurrency after applying the current SlowDown reduction. Always ≥ 1
|
||||
/// so maintenance can still make progress under sustained rate pressure.
|
||||
pub(crate) fn effective_concurrency(&self) -> usize {
|
||||
let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
|
||||
(self.base_concurrency >> step).max(1)
|
||||
}
|
||||
|
||||
/// Fold one finished object into the rate state. Call once per outcome.
|
||||
/// `saw_slow_down` is true when any attempt of this object hit SlowDown,
|
||||
/// including attempts that later succeeded on retry.
|
||||
pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
|
||||
if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
|
||||
self.observe_slow_down();
|
||||
return;
|
||||
}
|
||||
if result.is_ok() {
|
||||
self.observe_success();
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
|
||||
fn observe_slow_down(&self) {
|
||||
// fetch_add returns the previous step; +1 so the first SlowDown uses
|
||||
// exponent 1 (2s), not exponent 0 (1s).
|
||||
let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
|
||||
let delay = Self::backoff(step);
|
||||
let ready = Instant::now() + delay;
|
||||
if let Ok(mut until) = self.slow_down_until.lock()
|
||||
&& until.is_none_or(|current| current < ready)
|
||||
{
|
||||
*until = Some(ready);
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore one concurrency step only after the SlowDown cool-down expires.
|
||||
/// A same-page success must not undo the reduction while the pass is still
|
||||
/// in the rate-limited window (the page semaphore is fixed until the next page).
|
||||
fn observe_success(&self) {
|
||||
let cooling = self
|
||||
.slow_down_until
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|until| *until)
|
||||
.is_some_and(|ready| ready > Instant::now());
|
||||
if cooling {
|
||||
return;
|
||||
}
|
||||
self.slow_down_step
|
||||
.try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
|
||||
.ok();
|
||||
}
|
||||
|
||||
/// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
|
||||
pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
|
||||
self.wait_slow_down(cancel).await?;
|
||||
if let Some(foreground) = &self.foreground {
|
||||
foreground.wait(cancel).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Foreground admission decision while the caller owns page capacity.
|
||||
pub(crate) fn admission_decision(&self) -> PacingDecision {
|
||||
match &self.foreground {
|
||||
Some(foreground) => foreground.admission_decision(),
|
||||
None => PacingDecision::Ready,
|
||||
}
|
||||
}
|
||||
|
||||
/// Foreground wait after a rejected admission. Returns whether a pause was paid.
|
||||
pub(crate) async fn wait_after_admission(
|
||||
&self,
|
||||
cancel: &CancellationToken,
|
||||
observed: Option<ForegroundPressure>,
|
||||
) -> Result<bool> {
|
||||
match &self.foreground {
|
||||
Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
|
||||
let delay = self
|
||||
.slow_down_until
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|until| *until)
|
||||
.map(|ready| ready.saturating_duration_since(Instant::now()));
|
||||
let Some(delay) = delay else {
|
||||
return Ok(());
|
||||
};
|
||||
if delay.is_zero() {
|
||||
return Ok(());
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => Err(Error::TaskCancelled),
|
||||
_ = tokio::time::sleep(delay) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn backoff(step: u32) -> Duration {
|
||||
// step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
|
||||
// rate with exponential backoff, and heal must do the same to itself.
|
||||
let exponent = step.clamp(1, 5);
|
||||
Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct TestPressure {
|
||||
pub(crate) active: std::sync::atomic::AtomicUsize,
|
||||
@@ -308,87 +164,12 @@ mod tests {
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_first_slow_down_waits_two_seconds() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
|
||||
let cancel = CancellationToken::new();
|
||||
let before = Instant::now();
|
||||
rate.wait_before_page(&cancel).await.expect("cool-down wait");
|
||||
assert_eq!(Instant::now() - before, Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_reduces_concurrency_and_backs_off() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
assert_eq!(rate.effective_concurrency(), 8);
|
||||
rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
|
||||
assert_eq!(rate.effective_concurrency(), 2);
|
||||
// Floor is one so maintenance never stops entirely.
|
||||
for _ in 0..8 {
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
|
||||
}
|
||||
assert_eq!(rate.effective_concurrency(), 1);
|
||||
|
||||
let cancel = CancellationToken::new();
|
||||
assert!(rate.wait_before_page(&cancel).await.is_ok());
|
||||
|
||||
// After cool-down, successes restore concurrency one step at a time.
|
||||
for _ in 0..12 {
|
||||
rate.observe_object_outcome(&Ok(true), false);
|
||||
}
|
||||
assert!(rate.effective_concurrency() > 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
// Same-page successes while cool-down is armed must not restore fan-out.
|
||||
for _ in 0..4 {
|
||||
rate.observe_object_outcome(&Ok(true), false);
|
||||
}
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
// Replacement retry: SlowDown on attempt 1, success on attempt 2.
|
||||
rate.observe_object_outcome(&Ok(true), true);
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_wait_is_cancellable() {
|
||||
let rate = HealPageRate::new(4, None);
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
|
||||
let cancel = CancellationToken::new();
|
||||
cancel.cancel();
|
||||
assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
|
||||
let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
|
||||
.expect("foreground pacer");
|
||||
let rate = HealPageRate::new(4, Some(foreground));
|
||||
let cancel = CancellationToken::new();
|
||||
let before = Instant::now();
|
||||
rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
|
||||
assert_eq!(Instant::now() - before, Duration::from_millis(250));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
|
||||
for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
|
||||
let provider = Arc::new(TestPressure::new(class, 0));
|
||||
let pause = Duration::from_millis(250);
|
||||
let pacer = ForegroundPacing::new(
|
||||
let pacer = MainlinePacer::new(
|
||||
provider.clone(),
|
||||
if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
|
||||
if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
|
||||
@@ -416,9 +197,9 @@ mod tests {
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
|
||||
assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
|
||||
assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
|
||||
let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
|
||||
assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
|
||||
assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
|
||||
let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
|
||||
let before = Instant::now();
|
||||
pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
|
||||
assert_eq!(Instant::now() - before, Duration::from_secs(1));
|
||||
@@ -428,7 +209,7 @@ mod tests {
|
||||
async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
|
||||
let paced = Arc::new(
|
||||
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
|
||||
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
|
||||
);
|
||||
let cancel_first = CancellationToken::new();
|
||||
let first = tokio::spawn({
|
||||
@@ -447,7 +228,7 @@ mod tests {
|
||||
cancel_second.cancel();
|
||||
assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
|
||||
provider.active.store(0, Ordering::SeqCst);
|
||||
let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
|
||||
let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
|
||||
.expect("independent task");
|
||||
let before = Instant::now();
|
||||
other_task
|
||||
|
||||
@@ -35,9 +35,7 @@ mod replacement;
|
||||
mod utils;
|
||||
|
||||
pub(crate) use checkpoint::AdminErasureCheckpoint;
|
||||
pub use checkpoint::{
|
||||
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
|
||||
};
|
||||
pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
|
||||
pub(crate) use gc::ResumeGc;
|
||||
pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
|
||||
pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};
|
||||
|
||||
@@ -152,42 +152,6 @@ impl AdminErasureCheckpoint {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lightweight progress view for resume decisions. Omits the per-page object
|
||||
/// ledgers so callers do not clone a million-object checkpoint just to read
|
||||
/// counters and positions.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct CheckpointProgressSnapshot {
|
||||
pub current_bucket_index: usize,
|
||||
pub current_object_index: usize,
|
||||
pub successful_objects: u64,
|
||||
pub failed_object_count: u64,
|
||||
pub skipped_object_count: u64,
|
||||
pub skipped_new_versions: u64,
|
||||
pub skipped_ilm_expired: u64,
|
||||
pub processed_bytes: u64,
|
||||
pub total_objects: u64,
|
||||
pub total_bytes: u64,
|
||||
pub baseline_generation: Option<u64>,
|
||||
pub baseline_known: bool,
|
||||
pub counter_unknown: bool,
|
||||
}
|
||||
|
||||
impl CheckpointProgressSnapshot {
|
||||
pub fn has_progress(&self) -> bool {
|
||||
self.baseline_known
|
||||
|| self.successful_objects > 0
|
||||
|| self.failed_object_count > 0
|
||||
|| self.skipped_object_count > 0
|
||||
|| self.skipped_new_versions > 0
|
||||
|| self.skipped_ilm_expired > 0
|
||||
|| self.processed_bytes > 0
|
||||
|| self.total_objects > 0
|
||||
|| self.total_bytes > 0
|
||||
|| self.baseline_generation.is_some()
|
||||
|| self.counter_unknown
|
||||
}
|
||||
}
|
||||
|
||||
/// resume checkpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResumeCheckpoint {
|
||||
@@ -751,75 +715,6 @@ impl CheckpointManager {
|
||||
self.checkpoint.read().await.clone()
|
||||
}
|
||||
|
||||
/// True when a page object is already settled in the current ledger.
|
||||
/// Membership is checked under the lock without cloning the checkpoint,
|
||||
/// which used to copy every page's dedup sets on the small-object path.
|
||||
pub async fn is_object_settled(&self, key: &str) -> bool {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
checkpoint.processed_objects.contains(key)
|
||||
|| checkpoint.failed_objects.contains(key)
|
||||
|| checkpoint.skipped_objects.contains(key)
|
||||
}
|
||||
|
||||
/// True when the administrator checkpoint already finished.
|
||||
pub async fn admin_completed(&self) -> bool {
|
||||
self.checkpoint
|
||||
.read()
|
||||
.await
|
||||
.admin
|
||||
.as_ref()
|
||||
.is_some_and(|admin| admin.completed)
|
||||
}
|
||||
|
||||
/// Current resume position without cloning the object ledgers.
|
||||
pub async fn resume_position(&self) -> (usize, usize) {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
(checkpoint.current_bucket_index, checkpoint.current_object_index)
|
||||
}
|
||||
|
||||
/// Dangling-delete grace summary without cloning the object ledgers.
|
||||
pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
(
|
||||
checkpoint.dangling_delete_grace_objects,
|
||||
checkpoint.dangling_delete_retry_not_before,
|
||||
checkpoint.counter_unknown,
|
||||
)
|
||||
}
|
||||
|
||||
/// Aggregate progress counters without cloning the page dedup sets.
|
||||
pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
CheckpointProgressSnapshot {
|
||||
current_bucket_index: checkpoint.current_bucket_index,
|
||||
current_object_index: checkpoint.current_object_index,
|
||||
successful_objects: checkpoint.successful_objects,
|
||||
failed_object_count: checkpoint.failed_object_count,
|
||||
skipped_object_count: checkpoint.skipped_object_count,
|
||||
skipped_new_versions: checkpoint.skipped_new_versions,
|
||||
skipped_ilm_expired: checkpoint.skipped_ilm_expired,
|
||||
processed_bytes: checkpoint.processed_bytes,
|
||||
total_objects: checkpoint.total_objects,
|
||||
total_bytes: checkpoint.total_bytes,
|
||||
baseline_generation: checkpoint.baseline_generation,
|
||||
baseline_known: checkpoint.baseline_known,
|
||||
counter_unknown: checkpoint.counter_unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Finish the administrator outcome in place and persist it.
|
||||
pub async fn complete_admin(&self) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
let admin = checkpoint
|
||||
.admin
|
||||
.as_mut()
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
|
||||
admin.completed = true;
|
||||
admin.outcome.finish(None);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint().await
|
||||
}
|
||||
|
||||
/// update position
|
||||
pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
|
||||
@@ -485,7 +485,7 @@ pub struct HealTask {
|
||||
pub cancel_token: tokio_util::sync::CancellationToken,
|
||||
/// Storage layer interface
|
||||
pub storage: Arc<dyn HealStorageAPI>,
|
||||
mainline_pacer: Option<super::pacing::ForegroundPacing>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
@@ -586,7 +586,7 @@ impl HealTask {
|
||||
task
|
||||
}
|
||||
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
|
||||
self.mainline_pacer = pacer;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -480,9 +480,8 @@ impl HealTask {
|
||||
message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
|
||||
})?;
|
||||
if current_bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::StaleBucketIncarnation {
|
||||
bucket: bucket.to_owned(),
|
||||
expected: Some(bucket_incarnation_id),
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
expected.bucket_incarnation_id = Some(bucket_incarnation_id);
|
||||
|
||||
@@ -3375,67 +3375,10 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
|
||||
|
||||
let result = task.heal_object("bucket", "object", None).await;
|
||||
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
|
||||
));
|
||||
assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
|
||||
assert!(storage.heal_object_calls.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
|
||||
for object in ["timed out", "network error", "ordinary"] {
|
||||
let original_incarnation = Uuid::new_v4();
|
||||
let recreated_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
|
||||
request.options.pool_index = Some(0);
|
||||
request.options.set_index = Some(1);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("a recreated bucket must reject the old MRF responsibility");
|
||||
assert!(
|
||||
!error.is_recoverable_heal(),
|
||||
"user-controlled object names must not make a stale incarnation retryable: {object}"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
&error,
|
||||
Error::StaleBucketIncarnation { bucket, expected }
|
||||
if bucket == "bucket" && *expected == Some(original_incarnation)
|
||||
),
|
||||
"the failure must retain the original bucket incarnation: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
|
||||
"stale responsibility must be classified independently of the object name"
|
||||
);
|
||||
assert!(
|
||||
storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
|
||||
"an old responsibility must not reach storage repair in the recreated bucket"
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
|
||||
assert!(
|
||||
outcome.objects.iter().all(|item| !matches!(
|
||||
&item.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)),
|
||||
"the stale task must not produce a positive repair proof"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
|
||||
+81
-387
@@ -15,6 +15,7 @@
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
ops::{Deref, DerefMut},
|
||||
ptr,
|
||||
sync::{Arc, Mutex, Weak},
|
||||
};
|
||||
|
||||
@@ -63,46 +64,21 @@ impl Default for CacheState {
|
||||
pub struct Cache {
|
||||
state: ArcSwap<CacheState>,
|
||||
write_lock: Mutex<()>,
|
||||
reload_lock: AsyncMutex<()>,
|
||||
reload_changes: Mutex<Weak<Mutex<ReloadChanges>>>,
|
||||
service_account_mutation_lock: AsyncMutex<()>,
|
||||
sts_account_mutation_locks: Arc<MutationLockRegistry>,
|
||||
parent_mutation_locks: Arc<MutationLockRegistry>,
|
||||
sts_account_mutation_locks: Arc<StsMutationLockRegistry>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MutationLockRegistry {
|
||||
locks: Mutex<HashMap<String, Weak<AsyncMutex<MutationLockState>>>>,
|
||||
struct StsMutationLockRegistry {
|
||||
locks: Mutex<HashMap<String, Weak<AsyncMutex<StsMutationLockState>>>>,
|
||||
}
|
||||
|
||||
pub(crate) struct MutationLockState {
|
||||
pub(crate) struct StsMutationLockState {
|
||||
access_key: String,
|
||||
registry: Weak<MutationLockRegistry>,
|
||||
lock: Weak<AsyncMutex<MutationLockState>>,
|
||||
registry: Weak<StsMutationLockRegistry>,
|
||||
lock: Weak<AsyncMutex<StsMutationLockState>>,
|
||||
}
|
||||
|
||||
impl MutationLockRegistry {
|
||||
fn lock_for(self: &Arc<Self>, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
|
||||
let mut locks = self.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let registry = Arc::downgrade(self);
|
||||
let access_key_owned = access_key.to_string();
|
||||
let lock = Arc::new_cyclic(|lock| {
|
||||
AsyncMutex::new(MutationLockState {
|
||||
access_key: access_key_owned,
|
||||
registry,
|
||||
lock: lock.clone(),
|
||||
})
|
||||
});
|
||||
locks.insert(access_key.to_string(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MutationLockState {
|
||||
impl Drop for StsMutationLockState {
|
||||
fn drop(&mut self) {
|
||||
let Some(registry) = self.registry.upgrade() else {
|
||||
return;
|
||||
@@ -119,52 +95,10 @@ impl Default for Cache {
|
||||
Self {
|
||||
state: ArcSwap::new(Arc::new(CacheState::default())),
|
||||
write_lock: Mutex::new(()),
|
||||
reload_lock: AsyncMutex::new(()),
|
||||
reload_changes: Mutex::new(Weak::new()),
|
||||
service_account_mutation_lock: AsyncMutex::new(()),
|
||||
sts_account_mutation_locks: Arc::new(MutationLockRegistry::default()),
|
||||
parent_mutation_locks: Arc::new(MutationLockRegistry::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Hash, PartialEq, Eq)]
|
||||
pub(crate) enum CachePartition {
|
||||
PolicyDocs,
|
||||
Users,
|
||||
UserPolicies,
|
||||
StsAccounts,
|
||||
StsPolicies,
|
||||
Groups,
|
||||
UserGroupMemberships,
|
||||
GroupPolicies,
|
||||
}
|
||||
|
||||
type ReloadChanges = HashMap<CachePartition, HashSet<String>>;
|
||||
|
||||
/// A reload journals only keys changed during its own lifetime, including
|
||||
/// deletions. Unrelated STS traffic cannot suppress a persisted revocation.
|
||||
pub(crate) struct CacheReload<'a> {
|
||||
_guard: tokio::sync::MutexGuard<'a, ()>,
|
||||
changes: Arc<Mutex<ReloadChanges>>,
|
||||
}
|
||||
|
||||
impl CacheReload<'_> {
|
||||
pub(crate) fn merge_partition<T: Clone>(
|
||||
&self,
|
||||
partition: CachePartition,
|
||||
loaded: &mut CacheEntity<T>,
|
||||
current: &CacheEntity<T>,
|
||||
) {
|
||||
let changes = self.changes.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(keys) = changes.get(&partition) {
|
||||
for key in keys {
|
||||
if let Some(value) = current.get(key) {
|
||||
loaded.insert(key.clone(), value.clone());
|
||||
} else {
|
||||
loaded.remove(key);
|
||||
}
|
||||
}
|
||||
sts_account_mutation_locks: Arc::new(StsMutationLockRegistry {
|
||||
locks: Mutex::new(HashMap::new()),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -172,27 +106,31 @@ impl CacheReload<'_> {
|
||||
pub(crate) type CacheSnapshot = Guard<Arc<CacheState>>;
|
||||
|
||||
impl Cache {
|
||||
pub(crate) async fn begin_reload(&self) -> CacheReload<'_> {
|
||||
// Registration and publication both take write_lock before the journal
|
||||
// lock, so no mutation can fall between the reload and its journal.
|
||||
let guard = self.reload_lock.lock().await;
|
||||
let changes = Arc::new(Mutex::new(HashMap::new()));
|
||||
let _write = self.write_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
*self.reload_changes.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Arc::downgrade(&changes);
|
||||
CacheReload { _guard: guard, changes }
|
||||
}
|
||||
|
||||
pub(crate) fn service_account_mutation_lock(&self) -> &AsyncMutex<()> {
|
||||
&self.service_account_mutation_lock
|
||||
}
|
||||
|
||||
pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
|
||||
self.sts_account_mutation_locks.lock_for(access_key)
|
||||
}
|
||||
pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<StsMutationLockState>> {
|
||||
let mut locks = self
|
||||
.sts_account_mutation_locks
|
||||
.locks
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
/// Orders derived-credential publication with changes to its regular parent.
|
||||
pub(crate) fn parent_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<MutationLockState>> {
|
||||
self.parent_mutation_locks.lock_for(access_key)
|
||||
let registry = Arc::downgrade(&self.sts_account_mutation_locks);
|
||||
let access_key_owned = access_key.to_string();
|
||||
let lock = Arc::new_cyclic(|lock| {
|
||||
AsyncMutex::new(StsMutationLockState {
|
||||
access_key: access_key_owned,
|
||||
registry,
|
||||
lock: lock.clone(),
|
||||
})
|
||||
});
|
||||
locks.insert(access_key.to_string(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> CacheSnapshot {
|
||||
@@ -204,13 +142,7 @@ impl Cache {
|
||||
let current = self.state.load_full();
|
||||
let mut locked = LockedCache {
|
||||
state: CacheState::clone(¤t),
|
||||
#[cfg(test)]
|
||||
current_ptr: Arc::as_ptr(¤t),
|
||||
reload_changes: self
|
||||
.reload_changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.upgrade(),
|
||||
dirty: false,
|
||||
};
|
||||
let ret = f(&mut locked);
|
||||
@@ -283,9 +215,7 @@ impl Cache {
|
||||
|
||||
pub(crate) struct LockedCache {
|
||||
state: CacheState,
|
||||
#[cfg(test)]
|
||||
current_ptr: *const CacheState,
|
||||
reload_changes: Option<Arc<Mutex<ReloadChanges>>>,
|
||||
dirty: bool,
|
||||
}
|
||||
|
||||
@@ -294,19 +224,11 @@ impl LockedCache {
|
||||
&self.state
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn matches_snapshot(&self, snapshot: &CacheSnapshot) -> bool {
|
||||
std::ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
|
||||
ptr::eq(self.current_ptr, Arc::as_ptr(snapshot))
|
||||
}
|
||||
|
||||
fn exec<T: Clone>(
|
||||
target: &mut Arc<CacheEntity<T>>,
|
||||
t: OffsetDateTime,
|
||||
changes: Option<&Arc<Mutex<ReloadChanges>>>,
|
||||
partition: CachePartition,
|
||||
key: &str,
|
||||
mut op: impl FnMut(&mut CacheEntity<T>),
|
||||
) -> bool {
|
||||
fn exec<T: Clone>(target: &mut Arc<CacheEntity<T>>, t: OffsetDateTime, mut op: impl FnMut(&mut CacheEntity<T>)) -> bool {
|
||||
if target.load_time >= t {
|
||||
return false;
|
||||
}
|
||||
@@ -314,14 +236,6 @@ impl LockedCache {
|
||||
let mut new = CacheEntity::clone(target);
|
||||
op(&mut new);
|
||||
*target = Arc::new(new);
|
||||
if let Some(changes) = changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(partition)
|
||||
.or_default()
|
||||
.insert(key.to_string());
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
@@ -330,315 +244,139 @@ impl LockedCache {
|
||||
}
|
||||
|
||||
pub(crate) fn replace_policy_docs(&mut self, value: CacheEntity<PolicyDoc>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::PolicyDocs)
|
||||
.or_default()
|
||||
.extend(self.state.policy_docs.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.policy_docs = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_users(&mut self, value: CacheEntity<UserIdentity>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::Users)
|
||||
.or_default()
|
||||
.extend(self.state.users.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.users = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_user_policies(&mut self, value: CacheEntity<MappedPolicy>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::UserPolicies)
|
||||
.or_default()
|
||||
.extend(self.state.user_policies.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.user_policies = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_sts_accounts(&mut self, value: CacheEntity<UserIdentity>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::StsAccounts)
|
||||
.or_default()
|
||||
.extend(self.state.sts_accounts.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.sts_accounts = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_sts_policies(&mut self, value: CacheEntity<MappedPolicy>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::StsPolicies)
|
||||
.or_default()
|
||||
.extend(self.state.sts_policies.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.sts_policies = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_groups(&mut self, value: CacheEntity<GroupInfo>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::Groups)
|
||||
.or_default()
|
||||
.extend(self.state.groups.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.groups = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_group_policies(&mut self, value: CacheEntity<MappedPolicy>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::GroupPolicies)
|
||||
.or_default()
|
||||
.extend(self.state.group_policies.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.group_policies = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn replace_user_group_memberships(&mut self, value: CacheEntity<HashSet<String>>) {
|
||||
if let Some(changes) = &self.reload_changes {
|
||||
changes
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.entry(CachePartition::UserGroupMemberships)
|
||||
.or_default()
|
||||
.extend(self.state.user_group_memberships.keys().chain(value.keys()).cloned());
|
||||
}
|
||||
self.state.user_group_memberships = Self::replaced(value);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_policy_doc(&mut self, key: &str, value: &PolicyDoc, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.policy_docs,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::PolicyDocs,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_user(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.users,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::Users,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.users, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_user_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.user_policies,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::UserPolicies,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_sts_account(&mut self, key: &str, value: &UserIdentity, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.sts_accounts,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::StsAccounts,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_sts_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.sts_policies,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::StsPolicies,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_group(&mut self, key: &str, value: &GroupInfo, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.groups,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::Groups,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_user_group_membership(&mut self, key: &str, value: &HashSet<String>, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.user_group_memberships,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::UserGroupMemberships,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn add_or_update_group_policy(&mut self, key: &str, value: &MappedPolicy, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.group_policies,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::GroupPolicies,
|
||||
key,
|
||||
|map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
|
||||
map.insert(key.to_string(), value.clone());
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_policy_doc(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.policy_docs,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::PolicyDocs,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.policy_docs, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_user(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.users,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::Users,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.users, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_user_policy(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.user_policies,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::UserPolicies,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.user_policies, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_user_group_membership(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.user_group_memberships,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::UserGroupMemberships,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.user_group_memberships, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_sts_account(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.sts_accounts,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::StsAccounts,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.sts_accounts, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_sts_policy(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.sts_policies,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::StsPolicies,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.sts_policies, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_group(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.groups,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::Groups,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.groups, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn delete_group_policy(&mut self, key: &str, t: OffsetDateTime) {
|
||||
self.dirty |= Self::exec(
|
||||
&mut self.state.group_policies,
|
||||
t,
|
||||
self.reload_changes.as_ref(),
|
||||
CachePartition::GroupPolicies,
|
||||
key,
|
||||
|map| {
|
||||
map.remove(key);
|
||||
},
|
||||
);
|
||||
self.dirty |= Self::exec(&mut self.state.group_policies, t, |map| {
|
||||
map.remove(key);
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn build_user_group_memberships(&mut self) {
|
||||
@@ -756,50 +494,6 @@ mod tests {
|
||||
use crate::cache::Cache;
|
||||
use crate::store::MappedPolicy;
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_journal_preserves_same_key_mutations_and_expires_on_cancellation() {
|
||||
use super::{CacheEntity, CachePartition};
|
||||
let cache = Cache::default();
|
||||
let stale = UserIdentity {
|
||||
version: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let current = UserIdentity {
|
||||
version: 2,
|
||||
..Default::default()
|
||||
};
|
||||
let reload = cache.begin_reload().await;
|
||||
cache.add_or_update_user("updated", ¤t, OffsetDateTime::now_utc());
|
||||
cache.add_or_update_user("removed", ¤t, OffsetDateTime::now_utc());
|
||||
cache.delete_user("removed", OffsetDateTime::now_utc());
|
||||
// Even an absent key's deletion is recorded: storage may have returned
|
||||
// an entry created and deleted entirely during the reload's reads.
|
||||
cache.delete_user("absent", OffsetDateTime::now_utc());
|
||||
let mut loaded = CacheEntity::new(std::collections::HashMap::from([
|
||||
("updated".to_string(), stale.clone()),
|
||||
("removed".to_string(), stale.clone()),
|
||||
("absent".to_string(), stale.clone()),
|
||||
("disk-only".to_string(), stale),
|
||||
]));
|
||||
cache.with_write_lock(|locked| {
|
||||
reload.merge_partition(CachePartition::Users, &mut loaded, &locked.state().users);
|
||||
locked.replace_users(loaded);
|
||||
});
|
||||
let snapshot = cache.snapshot();
|
||||
assert_eq!(snapshot.users["updated"].version, 2);
|
||||
assert!(!snapshot.users.contains_key("removed"));
|
||||
assert!(!snapshot.users.contains_key("absent"));
|
||||
assert!(snapshot.users.contains_key("disk-only"));
|
||||
drop(reload);
|
||||
assert!(cache.reload_changes.lock().expect("journal lock").upgrade().is_none());
|
||||
let cancelled = cache.begin_reload().await;
|
||||
cache.delete_user("disk-only", OffsetDateTime::now_utc());
|
||||
drop(cancelled);
|
||||
assert!(cache.reload_changes.lock().expect("journal lock").upgrade().is_none());
|
||||
let next = cache.begin_reload().await;
|
||||
assert!(next.changes.lock().expect("new journal").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sts_mutation_locks_are_keyed_and_prune_unused_entries() {
|
||||
let cache = Cache::default();
|
||||
|
||||
@@ -16,7 +16,6 @@ use super::{FederatedClaims, VerifiedFederatedIdentity};
|
||||
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
|
||||
use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use serde_json::Value;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use tracing::debug;
|
||||
|
||||
@@ -67,15 +66,6 @@ impl FederatedAuthorization {
|
||||
&self.provider_id
|
||||
}
|
||||
|
||||
/// Issuer from the verified upstream identity.
|
||||
pub fn issuer(&self) -> Option<&str> {
|
||||
self.claims
|
||||
.raw
|
||||
.get("iss")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|issuer| !issuer.is_empty())
|
||||
}
|
||||
|
||||
pub fn subject(&self) -> &str {
|
||||
&self.claims.sub
|
||||
}
|
||||
|
||||
+7
-149
@@ -415,16 +415,6 @@ where
|
||||
|
||||
self.api.save_iam_config(IAMFormat::new_version_1(), path).await
|
||||
}
|
||||
/// Current built-in IAM group membership, independent of credential snapshots.
|
||||
pub fn user_group_memberships(&self, name: &str) -> Vec<String> {
|
||||
self.cache
|
||||
.snapshot()
|
||||
.user_group_memberships
|
||||
.get(name)
|
||||
.map(|groups| groups.iter().cloned().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub async fn get_user(&self, access_key: &str) -> Option<UserIdentity> {
|
||||
let cache = self.cache.snapshot();
|
||||
cache
|
||||
@@ -1015,17 +1005,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn sts_policy_db_get(&self, name: &str, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
|
||||
self.sts_policy_db_get_with_identity_mapping(name, groups)
|
||||
.await
|
||||
.map(|(policies, _)| policies)
|
||||
}
|
||||
|
||||
/// Resolve STS policies and whether that same lookup found a nonempty identity mapping.
|
||||
pub async fn sts_policy_db_get_with_identity_mapping(
|
||||
&self,
|
||||
name: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<(Vec<String>, bool)> {
|
||||
if name.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -1052,10 +1031,7 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
let direct_policies = mapped_policy.to_slice();
|
||||
let has_identity_mapping = !direct_policies.is_empty();
|
||||
let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
|
||||
Ok((policies, has_identity_mapping))
|
||||
self.policy_db_get_with_groups(mapped_policy.to_slice(), groups).await
|
||||
}
|
||||
|
||||
async fn policy_db_get_with_groups(&self, mut policies: Vec<String>, groups: &Option<Vec<String>>) -> Result<Vec<String>> {
|
||||
@@ -1182,7 +1158,7 @@ where
|
||||
if let Some(groups) = u.credentials.groups.as_ref() {
|
||||
for group in groups.iter() {
|
||||
if groups_cache.get(group).filter(|v| v.status == STATUS_DISABLED).is_some() {
|
||||
continue;
|
||||
return Ok((Vec::new(), OffsetDateTime::now_utc()));
|
||||
}
|
||||
|
||||
let mp = match group_policies.get(group) {
|
||||
@@ -1213,7 +1189,7 @@ where
|
||||
|
||||
for group in user_group_memberships.get(name).cloned().unwrap_or_default().iter() {
|
||||
if groups_cache.get(group).filter(|v| v.status == STATUS_DISABLED).is_some() {
|
||||
continue;
|
||||
return Ok((Vec::new(), OffsetDateTime::now_utc()));
|
||||
}
|
||||
|
||||
let mp = match group_policies.get(group) {
|
||||
@@ -1510,27 +1486,7 @@ where
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
.await
|
||||
.map(|(updated_at, _, _guard)| updated_at)
|
||||
}
|
||||
|
||||
/// Return whether this write replaced a different regular-user secret.
|
||||
pub(crate) async fn add_user_at_with_secret_change(
|
||||
&self,
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let parent_guard = parent_lock.lock_owned().await;
|
||||
let _service_guard = self.cache.service_account_mutation_lock().lock().await;
|
||||
let sts_lock = self.cache.sts_account_mutation_lock(access_key);
|
||||
let _sts_guard = sts_lock.lock().await;
|
||||
let cache = self.cache.snapshot();
|
||||
if cache.sts_accounts.contains_key(access_key) {
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
let users = Arc::clone(&cache.users);
|
||||
if let Some(x) = users.get(access_key) {
|
||||
debug!(
|
||||
@@ -1540,13 +1496,10 @@ where
|
||||
state = "existing",
|
||||
"Updating existing IAM user"
|
||||
);
|
||||
if x.credentials.is_temp() || x.credentials.is_service_account() {
|
||||
if x.credentials.is_temp() {
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
}
|
||||
let secret_changed = users
|
||||
.get(access_key)
|
||||
.is_some_and(|user| user.credentials.secret_key != args.secret_key);
|
||||
drop(cache);
|
||||
drop(users);
|
||||
|
||||
@@ -1570,7 +1523,7 @@ where
|
||||
|
||||
self.update_user_with_claims(access_key, user_entry)?;
|
||||
|
||||
Ok((updated_at, secret_changed, parent_guard))
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> {
|
||||
@@ -1587,17 +1540,12 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let parent_lock = (utype == UserType::Reg).then(|| self.cache.parent_mutation_lock(access_key));
|
||||
let _parent_guard = match &parent_lock {
|
||||
Some(lock) => Some(lock.lock().await),
|
||||
None => None,
|
||||
};
|
||||
let sts_mutation_lock = (utype == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(access_key));
|
||||
let _sts_mutation_guard = match &sts_mutation_lock {
|
||||
Some(lock) => Some(lock.lock().await),
|
||||
None => None,
|
||||
};
|
||||
let _service_account_guard = if matches!(utype, UserType::Svc | UserType::Reg) {
|
||||
let _service_account_guard = if utype == UserType::Svc {
|
||||
Some(self.cache.service_account_mutation_lock().lock().await)
|
||||
} else {
|
||||
None
|
||||
@@ -1692,46 +1640,17 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
|
||||
pub(crate) async fn update_user_secret_key_with_guard(
|
||||
&self,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>,
|
||||
)> {
|
||||
if access_key.is_empty() || secret_key.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let parent_guard = parent_lock.lock_owned().await;
|
||||
let cache = self.cache.snapshot();
|
||||
let users = Arc::clone(&cache.users);
|
||||
let u = match users.get(access_key) {
|
||||
Some(u) => u,
|
||||
None if expected_secret.is_some() => return Err(Error::IAMActionNotAllowed),
|
||||
None => return Err(Error::NoSuchUser(access_key.to_string())),
|
||||
};
|
||||
|
||||
if let Some(expected) = expected_secret {
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !u.credentials.is_valid()
|
||||
|| u.credentials.is_temp()
|
||||
|| u.credentials.is_service_account()
|
||||
|| !bool::from(u.credentials.secret_key.as_bytes().ct_eq(expected.as_bytes()))
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
}
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
@@ -1753,7 +1672,7 @@ where
|
||||
.await?;
|
||||
|
||||
self.update_user_with_claims(access_key, u)?;
|
||||
Ok((updated_at, status, parent_guard))
|
||||
Ok((updated_at, status))
|
||||
}
|
||||
|
||||
/// Add SSH public key for a user (for SFTP authentication)
|
||||
@@ -1762,8 +1681,6 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let _parent_guard = parent_lock.lock().await;
|
||||
let cache = self.cache.snapshot();
|
||||
let users = Arc::clone(&cache.users);
|
||||
let u = match users.get(access_key) {
|
||||
@@ -1803,8 +1720,6 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let _parent_guard = parent_lock.lock().await;
|
||||
let cache = self.cache.snapshot();
|
||||
let users = Arc::clone(&cache.users);
|
||||
let u = match users.get(access_key) {
|
||||
@@ -2323,11 +2238,6 @@ where
|
||||
Ok(())
|
||||
}
|
||||
pub async fn user_notification_handler(&self, name: &str, user_type: UserType) -> Result<()> {
|
||||
let parent_lock = (user_type == UserType::Reg).then(|| self.cache.parent_mutation_lock(name));
|
||||
let _parent_guard = match &parent_lock {
|
||||
Some(lock) => Some(lock.lock().await),
|
||||
None => None,
|
||||
};
|
||||
let sts_mutation_lock = (user_type == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(name));
|
||||
let _sts_mutation_guard = match &sts_mutation_lock {
|
||||
Some(lock) => Some(lock.lock().await),
|
||||
@@ -3075,58 +2985,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_group_preserves_direct_and_other_group_policies() {
|
||||
for credential_groups in [false, true] {
|
||||
let iam = build_test_iam_cache(FailingInitialLoadStore);
|
||||
let groups = vec!["disabled".to_string(), "enabled".to_string()];
|
||||
let user = UserIdentity::new(Credentials {
|
||||
access_key: "alice".to_string(),
|
||||
secret_key: "alice-secret".to_string(),
|
||||
status: STATUS_ENABLED.to_string(),
|
||||
groups: credential_groups.then(|| groups.clone()),
|
||||
..Default::default()
|
||||
});
|
||||
let now = OffsetDateTime::now_utc();
|
||||
iam.cache.add_or_update_user("alice", &user, now);
|
||||
iam.cache
|
||||
.add_or_update_user_policy("alice", &MappedPolicy::new("direct-deny"), now);
|
||||
if !credential_groups {
|
||||
iam.cache
|
||||
.add_or_update_user_group_membership("alice", &groups.iter().cloned().collect(), now);
|
||||
}
|
||||
for (group, status, policy) in [
|
||||
("disabled", STATUS_DISABLED, "disabled-grant"),
|
||||
("enabled", STATUS_ENABLED, "other-group-deny"),
|
||||
] {
|
||||
iam.cache.add_or_update_group(
|
||||
group,
|
||||
&GroupInfo {
|
||||
status: status.to_string(),
|
||||
members: vec!["alice".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
now,
|
||||
);
|
||||
iam.cache.add_or_update_group_policy(group, &MappedPolicy::new(policy), now);
|
||||
}
|
||||
let policies: HashSet<_> = iam
|
||||
.policy_db_get("alice", &None)
|
||||
.await
|
||||
.expect("user policy lookup")
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert_eq!(policies, HashSet::from(["direct-deny".to_string(), "other-group-deny".to_string()]));
|
||||
assert!(
|
||||
iam.policy_db_get_internal("disabled", true, false)
|
||||
.await
|
||||
.expect("disabled group lookup")
|
||||
.0
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_policy_lookup_ignores_colliding_regular_user_mapping() {
|
||||
let iam = build_test_iam_cache(FailingInitialLoadStore);
|
||||
|
||||
@@ -16,21 +16,19 @@ use super::{
|
||||
transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
|
||||
};
|
||||
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
|
||||
use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
|
||||
use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
|
||||
use openidconnect::{
|
||||
AsyncHttpClient, ClaimsVerificationError, ClientId, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
|
||||
ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
|
||||
AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::sync::RwLock;
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, warn};
|
||||
use url::Url;
|
||||
|
||||
pub(super) const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
|
||||
const OIDC_JWKS_REFRESH_COOLDOWN: StdDuration = StdDuration::from_secs(60);
|
||||
pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
|
||||
pub(super) const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
|
||||
pub(super) const OIDC_JWKS_BLOCKED_BY_OUTBOUND_POLICY: &str = "JWKS request blocked by outbound policy";
|
||||
@@ -98,14 +96,9 @@ pub(super) struct ProviderState {
|
||||
pub(super) discovered_at: Instant,
|
||||
}
|
||||
|
||||
type RefreshAttempt = Option<(Instant, Result<ProviderState, String>)>;
|
||||
type ProviderRefreshLock = Arc<tokio::sync::Mutex<RefreshAttempt>>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct ProviderRuntime {
|
||||
states: Arc<RwLock<HashMap<String, ProviderState>>>,
|
||||
http_client: Arc<ReqwestHttpClient>,
|
||||
refreshes: Arc<Mutex<HashMap<String, ProviderRefreshLock>>>,
|
||||
states: RwLock<HashMap<String, ProviderState>>,
|
||||
http_client: ReqwestHttpClient,
|
||||
}
|
||||
|
||||
// Workload issuers do not implement the browser authorization flow. Keep their
|
||||
@@ -206,57 +199,19 @@ impl DiscoveredProviderMetadata {
|
||||
|
||||
pub(super) fn verifier(&self, config: &OidcProviderConfig) -> CoreIdTokenVerifier<'static> {
|
||||
let client_id = ClientId::new(config.client_id.clone());
|
||||
let secret = config.client_secret.as_ref().map(|secret| ClientSecret::new(secret.clone()));
|
||||
let (issuer, jwks, algorithms) = match self {
|
||||
Self::Console(metadata) => (metadata.issuer(), metadata.jwks(), metadata.id_token_signing_alg_values_supported()),
|
||||
Self::Workload(metadata) => (&metadata.issuer, &metadata.jwks, &metadata.id_token_signing_alg_values_supported),
|
||||
};
|
||||
// Caller-supplied tokens must prove possession of the issuer's signing
|
||||
// key. A client secret is shared with clients and cannot provide that proof.
|
||||
let algorithms = algorithms
|
||||
.iter()
|
||||
.filter(|algorithm| {
|
||||
!matches!(
|
||||
algorithm,
|
||||
CoreJwsSigningAlgorithm::HmacSha256
|
||||
| CoreJwsSigningAlgorithm::HmacSha384
|
||||
| CoreJwsSigningAlgorithm::HmacSha512
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()).set_allowed_algs(algorithms)
|
||||
let verifier = match secret {
|
||||
Some(secret) => CoreIdTokenVerifier::new_confidential_client(client_id, secret, issuer.clone(), jwks.clone()),
|
||||
None => CoreIdTokenVerifier::new_public_client(client_id, issuer.clone(), jwks.clone()),
|
||||
};
|
||||
verifier.set_allowed_algs(algorithms.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
|
||||
token: &'a CoreIdToken,
|
||||
verifier: &CoreIdTokenVerifier<'_>,
|
||||
nonce_verifier: N,
|
||||
) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
|
||||
let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
|
||||
CoreJwsSigningAlgorithm::HmacSha256 => 32,
|
||||
CoreJwsSigningAlgorithm::HmacSha384 => 48,
|
||||
CoreJwsSigningAlgorithm::HmacSha512 => 64,
|
||||
_ => return token.claims(verifier, nonce_verifier),
|
||||
};
|
||||
// RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
|
||||
// openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
|
||||
let invalid_length = || {
|
||||
ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
|
||||
"invalid HMAC signature length".to_owned(),
|
||||
))
|
||||
};
|
||||
let compact = token.to_string();
|
||||
let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
|
||||
let signature = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(signature)
|
||||
.map_err(|_| invalid_length())?;
|
||||
if signature.len() != expected_length {
|
||||
return Err(invalid_length());
|
||||
}
|
||||
token.claims(verifier, nonce_verifier)
|
||||
}
|
||||
|
||||
// This adapter is used only for discovery/JWKS fetches, never token exchange.
|
||||
struct JwksAcceptClient<'a> {
|
||||
inner: &'a ReqwestHttpClient,
|
||||
@@ -288,9 +243,8 @@ impl ProviderState {
|
||||
impl ProviderRuntime {
|
||||
pub(super) fn new(http_client: ReqwestHttpClient, states: HashMap<String, ProviderState>) -> Self {
|
||||
Self {
|
||||
states: Arc::new(RwLock::new(states)),
|
||||
http_client: Arc::new(http_client),
|
||||
refreshes: Arc::new(Mutex::new(HashMap::new())),
|
||||
states: RwLock::new(states),
|
||||
http_client,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,38 +296,9 @@ impl ProviderRuntime {
|
||||
provider_id: &str,
|
||||
config: &OidcProviderConfig,
|
||||
) -> Result<ProviderState, String> {
|
||||
let refresh = self
|
||||
.refreshes
|
||||
.lock()
|
||||
.map_err(|_| "provider refresh registry is unavailable".to_string())?
|
||||
.entry(provider_id.to_string())
|
||||
.or_default()
|
||||
.clone();
|
||||
let mut attempt = refresh.lock_owned().await;
|
||||
if let Some((completed, result)) = attempt.as_ref()
|
||||
&& completed.elapsed() < OIDC_JWKS_REFRESH_COOLDOWN
|
||||
{
|
||||
return result.clone();
|
||||
}
|
||||
|
||||
// The admitted fetch owns the per-provider lock. A disconnected request
|
||||
// cannot abandon discovery while consuming the provider's cooldown.
|
||||
// Waiting callers share its completed result, including genuine failures.
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(|err| err.to_string())?;
|
||||
let provider_runtime = self.clone();
|
||||
let provider_id = provider_id.to_string();
|
||||
let config = config.clone();
|
||||
runtime
|
||||
.spawn(async move {
|
||||
let result = provider_runtime
|
||||
.recover_provider_state(&provider_id, &config)
|
||||
.await
|
||||
.map_err(|err| err.to_string());
|
||||
*attempt = Some((Instant::now(), result.clone()));
|
||||
result
|
||||
})
|
||||
self.recover_provider_state(provider_id, config)
|
||||
.await
|
||||
.map_err(|err| err.to_string())?
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(super) async fn recover_provider_state(
|
||||
|
||||
@@ -180,7 +180,7 @@ fn workload_metadata_rejects_duplicate_fields() {
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
|
||||
fn workload_verifier_preserves_algorithm_secret_and_audience_policy() {
|
||||
let secret = Nonce::new_random().secret().to_string();
|
||||
let mut config = build_mocked_oidc_provider_config("workload", "https://issuer.example.com");
|
||||
config.client_secret = Some(secret.clone());
|
||||
@@ -195,8 +195,7 @@ fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
|
||||
let token: CoreIdToken = signed.parse().unwrap();
|
||||
for workload in [false, true] {
|
||||
for (algorithms, accepted) in [
|
||||
(serde_json::json!(["RS256", "HS256"]), false),
|
||||
(serde_json::json!(["HS256", "unsupported-future-algorithm"]), false),
|
||||
(serde_json::json!(["HS256", "unsupported-future-algorithm"]), true),
|
||||
(serde_json::json!(["ES256"]), false),
|
||||
(serde_json::json!(["unsupported-future-algorithm"]), false),
|
||||
] {
|
||||
@@ -217,6 +216,21 @@ fn caller_supplied_tokens_cannot_be_signed_with_the_client_secret() {
|
||||
accepted,
|
||||
"workload={workload}, algorithms={algorithms}"
|
||||
);
|
||||
if accepted {
|
||||
assert!(
|
||||
token.claims(&metadata.verifier(&config), |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"additional audiences require explicit trust"
|
||||
);
|
||||
let mut wrong_secret = config.clone();
|
||||
wrong_secret.client_secret = Some(Nonce::new_random().secret().to_string());
|
||||
let verifier = metadata
|
||||
.verifier(&wrong_secret)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
assert!(
|
||||
token.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"incorrect client secret must fail"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -228,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let payload = serde_json::json!({
|
||||
"iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
|
||||
"iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
|
||||
"iat": now, "exp": now + 300,
|
||||
});
|
||||
|
||||
for (algorithm, name, signature_length) in [
|
||||
@@ -261,41 +275,22 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
|
||||
.expect("parse configured provider metadata");
|
||||
let verifier = metadata.verifier(&config);
|
||||
assert!(
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"caller-supplied HMAC tokens must be rejected"
|
||||
);
|
||||
// The authorization-code flow obtains tokens directly from the
|
||||
// provider and still supports confidential-client verification.
|
||||
let verifier = openidconnect::core::CoreIdTokenVerifier::new_confidential_client(
|
||||
openidconnect::ClientId::new(config.client_id.clone()),
|
||||
openidconnect::ClientSecret::new(secret.to_owned()),
|
||||
metadata.issuer().clone(),
|
||||
openidconnect::core::CoreJsonWebKeySet::new(vec![]),
|
||||
)
|
||||
.set_allowed_algs(vec![serde_json::from_value(serde_json::json!(name)).expect("parse signing algorithm")]);
|
||||
assert!(
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
"valid {name} token must pass for workload={workload}"
|
||||
);
|
||||
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
|
||||
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
|
||||
|
||||
for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
|
||||
let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
|
||||
let malformed: CoreIdToken = format!("{signing_input}.{signature}")
|
||||
.parse()
|
||||
.expect("invalid signature length must remain parseable before verification");
|
||||
.expect("invalid signature length must still reach cryptographic verification");
|
||||
assert!(
|
||||
verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"{name} signature length {length} must return an error for workload={workload}"
|
||||
);
|
||||
assert!(
|
||||
verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
|
||||
"{name} signature length {length} must return an error with a nonce for workload={workload}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
"invalid inputs must not prevent later valid {name} verification"
|
||||
);
|
||||
}
|
||||
@@ -779,149 +774,3 @@ async fn test_validate_oidc_provider_config_returns_detailed_errors() {
|
||||
assert!(err.contains(&format!("issuer '{base}/application/o/rustfs/'")));
|
||||
assert!(handle.join().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn jwks_refresh_is_single_flight_and_survives_request_cancellation() {
|
||||
let (initial_key, initial_jwk) = oidc_es256_key_and_jwk("initial");
|
||||
let (rotated_key, rotated_jwk) = oidc_es256_key_and_jwk("rotated");
|
||||
let (wrong_key, _) = oidc_es256_key_and_jwk("wrong");
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let started = Arc::new(tokio::sync::Notify::new());
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let server_requests = Arc::clone(&requests);
|
||||
let server_started = Arc::clone(&started);
|
||||
let Some((base, server)) = start_mock_oidc_discovery_server_with_jwks(
|
||||
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
|
||||
6,
|
||||
"ES256",
|
||||
false,
|
||||
move |fetch| {
|
||||
let request = server_requests.fetch_add(1, Ordering::SeqCst);
|
||||
if request == 2 {
|
||||
server_started.notify_one();
|
||||
release_rx
|
||||
.recv_timeout(StdDuration::from_secs(5))
|
||||
.expect("release refresh response");
|
||||
}
|
||||
serde_json::json!({"keys": [if fetch == 0 { &initial_jwk } else { &rotated_jwk }]}).to_string()
|
||||
},
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let mut config = build_mocked_oidc_provider_config("bounded-refresh", &base);
|
||||
config.issuer = Some(base.clone());
|
||||
let http = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).expect("allow test origin"));
|
||||
let state = discover_provider(&config, &http).await.expect("initial discovery");
|
||||
let sys = Arc::new(OidcSys {
|
||||
configs: HashMap::from([(config.id.clone(), test_sourced_config(config.clone()))]),
|
||||
provider_runtime: ProviderRuntime::new(http, HashMap::from([(config.id.clone(), state)])),
|
||||
state_store: OidcStateStore::new(),
|
||||
pending_discovery: Default::default(),
|
||||
});
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let payload = serde_json::json!({"iss": base, "sub": "user", "aud": config.client_id, "iat": now, "exp": now + 300});
|
||||
let sign = |kid: &str, claims: &serde_json::Value, key: &EncodingKey| {
|
||||
let mut header = Header::new(Algorithm::ES256);
|
||||
header.kid = Some(kid.to_string());
|
||||
jsonwebtoken::encode(&header, claims, key).expect("sign test token")
|
||||
};
|
||||
for (claim, value) in [
|
||||
("aud", serde_json::json!("wrong-audience")),
|
||||
("exp", serde_json::json!(now - 300)),
|
||||
] {
|
||||
let mut invalid = payload.clone();
|
||||
invalid[claim] = value;
|
||||
assert!(
|
||||
sys.verify_web_identity_token(&sign("initial", &invalid, &initial_key))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2, "claim failures cannot refresh signing keys");
|
||||
let invalid = sign("rotated", &payload, &wrong_key);
|
||||
let attack = {
|
||||
let sys = Arc::clone(&sys);
|
||||
let invalid = invalid.clone();
|
||||
tokio::spawn(async move { sys.verify_web_identity_token(&invalid).await })
|
||||
};
|
||||
tokio::time::timeout(StdDuration::from_secs(5), started.notified())
|
||||
.await
|
||||
.expect("refresh started");
|
||||
attack.abort();
|
||||
assert!(attack.await.expect_err("request canceled").is_cancelled());
|
||||
let mut retries = Vec::new();
|
||||
for _ in 0..16 {
|
||||
let sys = Arc::clone(&sys);
|
||||
let invalid = invalid.clone();
|
||||
retries.push(tokio::spawn(async move { sys.verify_web_identity_token(&invalid).await }));
|
||||
}
|
||||
release_tx.send(()).expect("release provider");
|
||||
for retry in retries {
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_secs(5), retry)
|
||||
.await
|
||||
.expect("retry timeout")
|
||||
.expect("retry join")
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let rotated = sign("rotated", &payload, &rotated_key);
|
||||
sys.verify_web_identity_token(&rotated)
|
||||
.await
|
||||
.expect("completed refresh must accept rotated keys after the original caller left");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4, "one discovery/JWKS pair for all rejected tokens");
|
||||
sys.provider_runtime
|
||||
.refresh_provider_state("independent-provider", &config)
|
||||
.await
|
||||
.expect("provider cooldowns must be independent");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 6);
|
||||
server.join().expect("mock server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn jwks_refresh_cools_down_failures_and_recovers_after_the_interval() {
|
||||
let (_, initial_jwk) = oidc_es256_key_and_jwk("initial");
|
||||
let (_, recovered_jwk) = oidc_es256_key_and_jwk("recovered");
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let server_requests = Arc::clone(&requests);
|
||||
let Some((base, server)) = start_mock_oidc_discovery_server_with_jwks(
|
||||
|base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()),
|
||||
6,
|
||||
"ES256",
|
||||
false,
|
||||
move |fetch| {
|
||||
server_requests.fetch_add(1, Ordering::SeqCst);
|
||||
match fetch {
|
||||
0 => serde_json::json!({"keys": [initial_jwk]}).to_string(),
|
||||
1 => "invalid JWKS".to_string(),
|
||||
_ => serde_json::json!({"keys": [recovered_jwk]}).to_string(),
|
||||
}
|
||||
},
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let mut config = build_mocked_oidc_provider_config("recovering-refresh", &base);
|
||||
config.issuer = Some(base.clone());
|
||||
let http = ReqwestHttpClient::with_policy(OutboundPolicy::from_allowed_origins(&base).expect("allow test origin"));
|
||||
let initial = discover_provider(&config, &http).await.expect("initial discovery");
|
||||
let runtime = ProviderRuntime::new(http, HashMap::from([(config.id.clone(), initial.clone())]));
|
||||
assert!(runtime.refresh_provider_state(&config.id, &config).await.is_err());
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
for _ in 0..8 {
|
||||
assert!(runtime.refresh_provider_state(&config.id, &config).await.is_err());
|
||||
}
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4, "completed failures also consume a bounded cooldown");
|
||||
assert_eq!(
|
||||
runtime.get_provider_state(&config.id).expect("cached keys").discovered_at,
|
||||
initial.discovered_at
|
||||
);
|
||||
let refresh = runtime.refreshes.lock().expect("registry")[&config.id].clone();
|
||||
refresh.lock().await.as_mut().expect("completed attempt").0 = Instant::now() - OIDC_JWKS_REFRESH_COOLDOWN;
|
||||
runtime
|
||||
.refresh_provider_state(&config.id, &config)
|
||||
.await
|
||||
.expect("retry after cooldown");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 6);
|
||||
assert!(runtime.get_provider_state(&config.id).expect("recovered keys").discovered_at > initial.discovered_at);
|
||||
server.join().expect("mock server");
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::{
|
||||
OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
|
||||
SourcedOidcProviderConfig, load_oidc_config_snapshot,
|
||||
},
|
||||
provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
|
||||
provider::{ProviderRuntime, discover_provider_for_recovery},
|
||||
state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
|
||||
transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
|
||||
};
|
||||
@@ -33,8 +33,8 @@ use crate::{
|
||||
};
|
||||
use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken};
|
||||
use openidconnect::{
|
||||
Audience, AuthType, AuthorizationCode, ClaimsVerificationError, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce,
|
||||
PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope,
|
||||
Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, LogoutRequest, Nonce, PkceCodeChallenge,
|
||||
PkceCodeVerifier, PostLogoutRedirectUrl, RedirectUrl, RequestTokenError, Scope,
|
||||
};
|
||||
use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
|
||||
use rustfs_utils::logging::LogThrottle;
|
||||
@@ -535,11 +535,8 @@ impl OidcSys {
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
|
||||
let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
|
||||
if let Err(e) = verified {
|
||||
if !matches!(e, ClaimsVerificationError::SignatureVerification(_)) {
|
||||
return Err(format!("ID token verification failed: {e}"));
|
||||
}
|
||||
let verification_error = e.to_string();
|
||||
warn!(
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
@@ -586,7 +583,8 @@ impl OidcSys {
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
|
||||
id_token
|
||||
.claims(&verifier, &Nonce::new(session.nonce.clone()))
|
||||
.map_err(|retry_err| {
|
||||
error!(
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
@@ -769,10 +767,7 @@ impl OidcSys {
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
if !matches!(e, ClaimsVerificationError::SignatureVerification(_)) {
|
||||
return Err(format!("ID token verification failed: {e}"));
|
||||
}
|
||||
if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
state = self
|
||||
.provider_runtime
|
||||
.refresh_provider_state(&provider_id, &config)
|
||||
@@ -785,7 +780,8 @@ impl OidcSys {
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
|
||||
id_token
|
||||
.claims(&verifier, |_: Option<&Nonce>| Ok(()))
|
||||
.map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,20 +14,6 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) fn is_unreadable_bucket_tags(error: &crate::error::Error) -> bool {
|
||||
let crate::error::Error::Io(error) = error else {
|
||||
return false;
|
||||
};
|
||||
let Some(storage_error) = error.get_ref().and_then(|inner| inner.downcast_ref::<IamStorageError>()) else {
|
||||
return false;
|
||||
};
|
||||
rustfs_ecstore::api::bucket::metadata::unreadable_config_refusal(storage_error)
|
||||
.is_some_and(|failure| failure.config_file == rustfs_ecstore::api::bucket::metadata::BUCKET_TAGGING_CONFIG)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::UnreadableBucketConfig;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::config::RUSTFS_CONFIG_PREFIX as IAM_CONFIG_ROOT_PREFIX;
|
||||
use rustfs_ecstore::api::config::com::{
|
||||
delete_config as ecstore_delete_config, read_config_no_lock as ecstore_read_config_no_lock,
|
||||
|
||||
+37
-146
@@ -1154,7 +1154,7 @@ impl Store for ObjectStore {
|
||||
// node-counted lock quorum (rustfs#4304; startup contract rustfs#4056).
|
||||
const LOAD_ALL_MODE: LoadMode = LoadMode::BootstrapNoLock;
|
||||
|
||||
let reload = cache.begin_reload().await;
|
||||
let cache_snapshot = cache.snapshot();
|
||||
let listed_config_items = self.list_all_iamconfig_items().await?;
|
||||
|
||||
let mut policy_docs_cache = CacheEntity::new(get_default_policies());
|
||||
@@ -1201,20 +1201,26 @@ impl Store for ObjectStore {
|
||||
}
|
||||
|
||||
// groups
|
||||
let mut groups_cache = CacheEntity::default();
|
||||
let mut groups_cache = None;
|
||||
if let Some(item_name_list) = listed_config_items.get(GROUPS_LIST_KEY) {
|
||||
let mut items_cache = CacheEntity::default();
|
||||
|
||||
for item in item_name_list.iter() {
|
||||
let name = rustfs_utils::path::dir(item);
|
||||
debug!(group = %name, "IAM group loaded");
|
||||
if let Err(err) = self.load_group_with(&name, &mut groups_cache, LOAD_ALL_MODE).await {
|
||||
if let Err(err) = self.load_group_with(&name, &mut items_cache, LOAD_ALL_MODE).await {
|
||||
return Err(Error::other(format!("load group failed: {err}")));
|
||||
};
|
||||
}
|
||||
|
||||
groups_cache = Some(items_cache);
|
||||
}
|
||||
|
||||
// user policies
|
||||
let mut user_policies_cache = CacheEntity::default();
|
||||
let mut user_policies_cache = None;
|
||||
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_USERS_LIST_KEY) {
|
||||
let mut items_cache = CacheEntity::default();
|
||||
|
||||
// Load in fixed-size chunks so each mapping is fetched exactly once
|
||||
// (backlog#806 — the split_off loop was O(n^2)).
|
||||
for chunk in item_name_list.chunks(32) {
|
||||
@@ -1229,26 +1235,32 @@ impl Store for ObjectStore {
|
||||
|
||||
let name = chunk[idx].trim_end_matches(".json").to_owned();
|
||||
debug!(user = %name, "IAM user policy loaded");
|
||||
user_policies_cache.insert(name, p);
|
||||
items_cache.insert(name, p);
|
||||
}
|
||||
}
|
||||
|
||||
user_policies_cache = Some(items_cache);
|
||||
}
|
||||
|
||||
// group policy
|
||||
let mut group_policies_cache = CacheEntity::default();
|
||||
let mut group_policies_cache = None;
|
||||
if let Some(item_name_list) = listed_config_items.get(POLICY_DB_GROUPS_LIST_KEY) {
|
||||
let mut items_cache = CacheEntity::default();
|
||||
|
||||
for item in item_name_list.iter() {
|
||||
let name = item.trim_end_matches(".json");
|
||||
|
||||
debug!(group = %name, "IAM group policy loaded");
|
||||
if let Err(err) = self
|
||||
.load_mapped_policy_with(name, UserType::Reg, true, &mut group_policies_cache, LOAD_ALL_MODE)
|
||||
.load_mapped_policy_with(name, UserType::Reg, true, &mut items_cache, LOAD_ALL_MODE)
|
||||
.await
|
||||
&& !is_err_no_such_policy(&err)
|
||||
{
|
||||
return Err(Error::other(format!("load group policy failed: {err}")));
|
||||
};
|
||||
}
|
||||
|
||||
group_policies_cache = Some(items_cache);
|
||||
}
|
||||
|
||||
let mut sts_policies_cache = CacheEntity::default();
|
||||
@@ -1318,48 +1330,25 @@ impl Store for ObjectStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
if let Ok(barrier) = LOAD_ALL_COMMIT_BARRIER.try_with(std::sync::Arc::clone) {
|
||||
barrier.0.notify_one();
|
||||
barrier.1.notified().await;
|
||||
}
|
||||
|
||||
cache.with_write_lock(|cache| {
|
||||
reload.merge_partition(
|
||||
crate::cache::CachePartition::PolicyDocs,
|
||||
&mut policy_docs_cache,
|
||||
&cache.state().policy_docs,
|
||||
);
|
||||
reload.merge_partition(crate::cache::CachePartition::Groups, &mut groups_cache, &cache.state().groups);
|
||||
reload.merge_partition(
|
||||
crate::cache::CachePartition::UserPolicies,
|
||||
&mut user_policies_cache,
|
||||
&cache.state().user_policies,
|
||||
);
|
||||
reload.merge_partition(
|
||||
crate::cache::CachePartition::GroupPolicies,
|
||||
&mut group_policies_cache,
|
||||
&cache.state().group_policies,
|
||||
);
|
||||
reload.merge_partition(crate::cache::CachePartition::Users, &mut user_items_cache, &cache.state().users);
|
||||
reload.merge_partition(
|
||||
crate::cache::CachePartition::StsAccounts,
|
||||
&mut sts_items_cache,
|
||||
&cache.state().sts_accounts,
|
||||
);
|
||||
reload.merge_partition(
|
||||
crate::cache::CachePartition::StsPolicies,
|
||||
&mut sts_policies_cache,
|
||||
&cache.state().sts_policies,
|
||||
);
|
||||
cache.replace_policy_docs(policy_docs_cache);
|
||||
cache.replace_groups(groups_cache);
|
||||
cache.replace_user_policies(user_policies_cache);
|
||||
cache.replace_group_policies(group_policies_cache);
|
||||
cache.replace_users(user_items_cache);
|
||||
cache.replace_sts_accounts(sts_items_cache);
|
||||
cache.replace_sts_policies(sts_policies_cache);
|
||||
cache.build_user_group_memberships();
|
||||
if cache.matches_snapshot(&cache_snapshot) {
|
||||
cache.replace_policy_docs(policy_docs_cache);
|
||||
if let Some(groups_cache) = groups_cache {
|
||||
cache.replace_groups(groups_cache);
|
||||
}
|
||||
if let Some(user_policies_cache) = user_policies_cache {
|
||||
cache.replace_user_policies(user_policies_cache);
|
||||
}
|
||||
if let Some(group_policies_cache) = group_policies_cache {
|
||||
cache.replace_group_policies(group_policies_cache);
|
||||
}
|
||||
cache.replace_users(user_items_cache);
|
||||
cache.replace_sts_accounts(sts_items_cache);
|
||||
cache.replace_sts_policies(sts_policies_cache);
|
||||
cache.build_user_group_memberships();
|
||||
} else {
|
||||
warn!("IAM full reload cache commit skipped due to concurrent cache changes");
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
@@ -1374,11 +1363,6 @@ fn map_delete_user_identity_error(name: &str, err: Error) -> Error {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static LOAD_ALL_COMMIT_BARRIER: std::sync::Arc<(tokio::sync::Notify, tokio::sync::Notify)>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{DecryptSource, LoadMode, ObjectStore, map_delete_user_identity_error};
|
||||
@@ -1388,99 +1372,6 @@ mod tests {
|
||||
use serial_test::serial;
|
||||
use temp_env::with_vars;
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_load_all_clears_partitions_with_empty_prefix() {
|
||||
use crate::cache::Cache;
|
||||
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
|
||||
std::thread::Builder::new()
|
||||
.stack_size(8 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(2)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build IAM reload test runtime");
|
||||
runtime.block_on(async {
|
||||
let _credentials = test_cred();
|
||||
let temp = tempfile::tempdir().expect("create IAM reload store directory");
|
||||
let env = Box::pin(
|
||||
rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.init_bucket_metadata(false)
|
||||
.build(),
|
||||
)
|
||||
.await;
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
store
|
||||
.save_group_info("reload-group", GroupInfo::new(vec!["reload-user".to_string()]))
|
||||
.await
|
||||
.expect("seed group membership");
|
||||
store
|
||||
.save_mapped_policy("reload-user", UserType::Reg, false, MappedPolicy::new("readwrite"), None)
|
||||
.await
|
||||
.expect("seed user policy");
|
||||
store
|
||||
.save_mapped_policy("reload-group", UserType::Reg, true, MappedPolicy::new("readonly"), None)
|
||||
.await
|
||||
.expect("seed group policy");
|
||||
|
||||
let cache = Cache::default();
|
||||
store.load_all(&cache).await.expect("load nonempty IAM partitions");
|
||||
{
|
||||
let snapshot = cache.snapshot();
|
||||
assert!(snapshot.groups.contains_key("reload-group"));
|
||||
assert_eq!(snapshot.user_policies["reload-user"].policies, "readwrite");
|
||||
assert_eq!(snapshot.group_policies["reload-group"].policies, "readonly");
|
||||
assert!(snapshot.user_group_memberships["reload-user"].contains("reload-group"));
|
||||
}
|
||||
|
||||
// Mutate durable state without notifying this cache, as on
|
||||
// a peer that missed the last mapping/group deletion.
|
||||
store
|
||||
.delete_mapped_policy("reload-user", UserType::Reg, false)
|
||||
.await
|
||||
.expect("remove last user mapping");
|
||||
store
|
||||
.delete_mapped_policy("reload-group", UserType::Reg, true)
|
||||
.await
|
||||
.expect("remove last group mapping");
|
||||
store.delete_group_info("reload-group").await.expect("remove last group");
|
||||
assert!(cache.snapshot().groups.contains_key("reload-group"));
|
||||
// Simulate a missed peer revocation while STS publication
|
||||
// and a separate user update overlap this full disk reload.
|
||||
let stale = rustfs_policy::auth::UserIdentity {
|
||||
version: 1,
|
||||
..Default::default()
|
||||
};
|
||||
cache.add_or_update_user("revoked-user", &stale, time::OffsetDateTime::now_utc());
|
||||
let barrier = std::sync::Arc::new((tokio::sync::Notify::new(), tokio::sync::Notify::new()));
|
||||
let loading = super::LOAD_ALL_COMMIT_BARRIER.scope(std::sync::Arc::clone(&barrier), store.load_all(&cache));
|
||||
let concurrent = async {
|
||||
barrier.0.notified().await;
|
||||
cache.add_or_update_sts_account("new-session", &stale, time::OffsetDateTime::now_utc());
|
||||
cache.add_or_update_user("new-service", &stale, time::OffsetDateTime::now_utc());
|
||||
barrier.1.notify_one();
|
||||
};
|
||||
let (loaded, ()) = tokio::join!(loading, concurrent);
|
||||
loaded.expect("reconcile revoked identities despite STS traffic");
|
||||
let snapshot = cache.snapshot();
|
||||
assert!(!snapshot.users.contains_key("revoked-user"));
|
||||
assert!(snapshot.users.contains_key("new-service"));
|
||||
assert!(snapshot.sts_accounts.contains_key("new-session"));
|
||||
assert!(snapshot.groups.is_empty());
|
||||
assert!(snapshot.user_policies.is_empty());
|
||||
assert!(snapshot.group_policies.is_empty());
|
||||
assert!(snapshot.user_group_memberships.is_empty());
|
||||
assert!(!snapshot.policy_docs.is_empty(), "built-in policies must remain available");
|
||||
});
|
||||
})
|
||||
.expect("spawn IAM reload test thread")
|
||||
.join()
|
||||
.expect("IAM reload test should complete");
|
||||
}
|
||||
|
||||
/// rustfs#4304 / startup contract rustfs#4056: the bulk snapshot load
|
||||
/// (`load_all`) must never depend on the node-counted namespace lock
|
||||
/// quorum, so its reads have to carry `no_lock = true` down to the
|
||||
|
||||
+18
-1013
File diff suppressed because it is too large
Load Diff
@@ -1411,28 +1411,6 @@ pub fn record_peer_reachable(addr: &str) {
|
||||
publish_offline_gauge(&peers);
|
||||
}
|
||||
|
||||
/// Record a successful interaction with a cluster peer, but only when the peer is currently
|
||||
/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
|
||||
/// and lock-path successes that do not go through a fresh dial.
|
||||
///
|
||||
/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
|
||||
/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
|
||||
/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
|
||||
pub fn record_peer_reachable_if_offline(addr: &str) {
|
||||
let key = normalize_peer_key(addr);
|
||||
// Fast path: read lock to check whether the peer is currently offline.
|
||||
{
|
||||
let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match peers.get(key) {
|
||||
None => return,
|
||||
Some(entry) if entry.online => return,
|
||||
Some(_) => {} // offline: fall through to the write path
|
||||
}
|
||||
}
|
||||
// Slow path: the peer is offline, so flip it back online and republish the gauge.
|
||||
record_peer_reachable(addr);
|
||||
}
|
||||
|
||||
/// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
|
||||
/// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
|
||||
pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
|
||||
@@ -2210,34 +2188,6 @@ mod tests {
|
||||
assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
|
||||
let addr = "http://cluster-peer-recover-if-offline-test:9000";
|
||||
|
||||
// Mark the peer offline.
|
||||
record_peer_unreachable(addr, 1);
|
||||
assert!(cluster_peer_is_offline(addr));
|
||||
|
||||
// A successful cached-channel RPC (no fresh dial) clears the offline state.
|
||||
record_peer_reachable_if_offline(addr);
|
||||
assert!(!cluster_peer_is_offline(addr));
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_peer_reachable_if_offline_no_op_for_online_peer() {
|
||||
let addr = "http://cluster-peer-recover-noop-test:9000";
|
||||
|
||||
record_peer_reachable(addr);
|
||||
// Should not panic or change state for an already-online peer.
|
||||
record_peer_reachable_if_offline(addr);
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
|
||||
// Should also be a no-op for unknown peers.
|
||||
record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
|
||||
assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_should_bypass_is_self_healing() {
|
||||
let addr = "http://cluster-peer-bypass-selfheal-test:9000";
|
||||
|
||||
@@ -85,17 +85,6 @@ impl KeystoneAuthProvider {
|
||||
Ok(self.keystone_token_to_credentials(&keystone_token))
|
||||
}
|
||||
|
||||
/// Revalidate a long-lived subscription directly with Keystone, bypassing
|
||||
/// the request cache so token revocation terminates an existing stream.
|
||||
/// Returns the same authentication and expiry errors as a new token login.
|
||||
pub async fn revalidate_subscription_token(&self, token: &str) -> Result<Credentials> {
|
||||
let validated = self.client.validate_token(token).await?;
|
||||
if validated.is_expired() {
|
||||
return Err(KeystoneError::TokenExpired);
|
||||
}
|
||||
Ok(self.keystone_token_to_credentials(&validated))
|
||||
}
|
||||
|
||||
/// Authenticate using EC2 credentials (S3 API with AWS SigV4)
|
||||
pub async fn authenticate_with_ec2(&self, access_key: &str, signature: &str, string_to_sign: &str) -> Result<Credentials> {
|
||||
// Check cache
|
||||
|
||||
@@ -121,7 +121,7 @@ impl KeystoneClient {
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("Failed to send token validation request: {}", e);
|
||||
KeystoneError::HttpError(e.without_url().to_string())
|
||||
KeystoneError::HttpError(e.to_string())
|
||||
})?;
|
||||
|
||||
let status = response.status();
|
||||
@@ -138,10 +138,7 @@ impl KeystoneClient {
|
||||
)));
|
||||
}
|
||||
|
||||
let body: serde_json::Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
|
||||
let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
|
||||
|
||||
self.parse_token_v3(&body)
|
||||
}
|
||||
@@ -263,16 +260,13 @@ impl KeystoneClient {
|
||||
.header("X-Auth-Token", admin_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| KeystoneError::HttpError(e.without_url().to_string()))?;
|
||||
.map_err(|e| KeystoneError::HttpError(e.to_string()))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let body: serde_json::Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
|
||||
let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
|
||||
|
||||
let credentials = body
|
||||
.get("credentials")
|
||||
@@ -371,7 +365,7 @@ impl KeystoneClient {
|
||||
.json(&auth_payload)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| KeystoneError::HttpError(e.without_url().to_string()))?;
|
||||
.map_err(|e| KeystoneError::HttpError(e.to_string()))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(KeystoneError::AuthenticationFailed("Admin authentication failed".to_string()));
|
||||
@@ -385,10 +379,7 @@ impl KeystoneClient {
|
||||
.to_string();
|
||||
|
||||
// Parse expiration from response body
|
||||
let body: serde_json::Value = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| KeystoneError::ParseError(e.without_url().to_string()))?;
|
||||
let body: serde_json::Value = response.json().await.map_err(|e| KeystoneError::ParseError(e.to_string()))?;
|
||||
|
||||
let expires_at = body
|
||||
.get("token")
|
||||
@@ -419,133 +410,6 @@ impl KeystoneClient {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscription_token_refresh_bypasses_a_still_valid_login_cache() {
|
||||
use std::io::{Read, Write};
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("fake Keystone");
|
||||
let endpoint = format!("http://{}", listener.local_addr().unwrap());
|
||||
let expires = (OffsetDateTime::now_utc() + time::Duration::hours(1))
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.unwrap();
|
||||
let body = json!({"token":{"user":{"id":"listener","name":"listener"},
|
||||
"roles":[{"name":"Member"}],"expires_at":expires}})
|
||||
.to_string();
|
||||
let server = std::thread::spawn(move || {
|
||||
for response in [
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
),
|
||||
"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
|
||||
] {
|
||||
let (mut socket, _) = listener.accept().expect("token validation request");
|
||||
socket.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap();
|
||||
let mut request = [0; 4096];
|
||||
let length = socket.read(&mut request).expect("request headers");
|
||||
assert!(String::from_utf8_lossy(&request[..length]).contains("subscription-token"));
|
||||
socket.write_all(response.as_bytes()).expect("validation response");
|
||||
}
|
||||
});
|
||||
let mut client = KeystoneClient::new(
|
||||
endpoint,
|
||||
KeystoneVersion::V3,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Default".into(),
|
||||
true,
|
||||
std::time::Duration::from_secs(5),
|
||||
);
|
||||
client.client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
let provider = crate::KeystoneAuthProvider::new(client, 10, std::time::Duration::from_secs(60), true);
|
||||
assert!(provider.authenticate_with_token("subscription-token").await.is_ok());
|
||||
assert!(
|
||||
provider.authenticate_with_token("subscription-token").await.is_ok(),
|
||||
"ordinary login still uses the valid cached token"
|
||||
);
|
||||
assert!(matches!(
|
||||
provider.revalidate_subscription_token("subscription-token").await,
|
||||
Err(KeystoneError::InvalidToken)
|
||||
));
|
||||
server.join().expect("fake Keystone finished");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn middleware_keystone_failures_do_not_disclose_upstream_locations() {
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use std::io::{Read, Write};
|
||||
use tower::{Layer, ServiceExt};
|
||||
|
||||
for (response, details) in [
|
||||
(None, "Keystone token validation failed"),
|
||||
(
|
||||
Some("HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nnot-json"),
|
||||
"Keystone token validation failed",
|
||||
),
|
||||
(
|
||||
Some("HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"),
|
||||
"Invalid token",
|
||||
),
|
||||
] {
|
||||
for header in ["X-Auth-Token", "X-Storage-Token"] {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind fake Keystone");
|
||||
let endpoint = format!("http://{}/private-deployment", listener.local_addr().unwrap());
|
||||
let server = response.map(|response| {
|
||||
std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("Keystone request");
|
||||
stream.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap();
|
||||
let mut request = [0; 4096];
|
||||
assert!(stream.read(&mut request).expect("read token validation request") > 0);
|
||||
stream.write_all(response.as_bytes()).expect("send fake response");
|
||||
})
|
||||
});
|
||||
let mut client = KeystoneClient::new(
|
||||
endpoint.clone(),
|
||||
KeystoneVersion::V3,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Default".to_owned(),
|
||||
true,
|
||||
std::time::Duration::from_secs(5),
|
||||
);
|
||||
client.client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
let provider = crate::KeystoneAuthProvider::new(client, 1, std::time::Duration::from_secs(1), false);
|
||||
let inner = tower::service_fn(|_| async {
|
||||
Ok::<_, std::convert::Infallible>(http::Response::new(Full::new(bytes::Bytes::new())))
|
||||
});
|
||||
let response = crate::middleware::KeystoneAuthLayer::new(Some(Arc::new(provider)))
|
||||
.layer(inner)
|
||||
.oneshot(
|
||||
http::Request::builder()
|
||||
.header(header, "private-token")
|
||||
.body(Full::new(bytes::Bytes::new()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), http::StatusCode::UNAUTHORIZED);
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let body = std::str::from_utf8(&body).unwrap();
|
||||
assert!(body.contains(&format!("<Details>{details}</Details>")));
|
||||
for secret in [&endpoint, "private-deployment", "private-token", "/v3/auth/tokens"] {
|
||||
assert!(!body.contains(secret), "upstream diagnostics must not appear in the response");
|
||||
}
|
||||
if let Some(server) = server {
|
||||
server.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_client_creation() {
|
||||
let client = KeystoneClient::new(
|
||||
|
||||
@@ -173,11 +173,7 @@ where
|
||||
<Message>Invalid Keystone token</Message>
|
||||
<Details>{}</Details>
|
||||
</Error>"#,
|
||||
xml_escape(match e {
|
||||
crate::KeystoneError::InvalidToken => "Invalid token",
|
||||
crate::KeystoneError::TokenExpired => "Token expired",
|
||||
_ => "Keystone token validation failed",
|
||||
})
|
||||
xml_escape(&e.to_string())
|
||||
);
|
||||
|
||||
let body: BoxBody = Full::new(Bytes::from(error_xml))
|
||||
|
||||
@@ -214,9 +214,6 @@ impl KmsManager {
|
||||
}
|
||||
|
||||
async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
if request.key_name.as_deref() == Some(crate::probe::PROBE_KEY_ID) {
|
||||
return Err(KmsError::invalid_operation("the KMS probe key is reserved for internal health checks"));
|
||||
}
|
||||
// A blank name is neither "generate one" (that is `None`) nor a usable
|
||||
// id: the Local backend would write a key file with an empty stem and
|
||||
// the Vault backends would address their mount root, each failing with
|
||||
@@ -441,9 +438,6 @@ impl KmsManager {
|
||||
/// any backend sees the request. The backends repeat the window bound as a
|
||||
/// defensive assertion for callers that hold a backend handle directly.
|
||||
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
if request.key_id == crate::probe::PROBE_KEY_ID {
|
||||
return Err(KmsError::invalid_operation("the KMS probe key cannot be deleted through key management"));
|
||||
}
|
||||
self.check_deletion_request(&request)?;
|
||||
if request.force_immediate.unwrap_or(false) {
|
||||
self.refuse_referenced_immediate_deletion(&request.key_id).await?;
|
||||
@@ -597,11 +591,7 @@ impl KmsManager {
|
||||
/// Disable a key on behalf of `context`'s principal
|
||||
pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
|
||||
let started = Instant::now();
|
||||
let result = if key_id == crate::probe::PROBE_KEY_ID {
|
||||
Err(KmsError::invalid_operation("the KMS probe key cannot be disabled through key management"))
|
||||
} else {
|
||||
self.backend.disable_key(key_id).await
|
||||
};
|
||||
let result = self.backend.disable_key(key_id).await;
|
||||
if result.is_ok() {
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
}
|
||||
|
||||
@@ -700,57 +700,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn key_management_cannot_preempt_or_disable_the_readiness_probe() {
|
||||
let temp_dir = tempfile::tempdir().expect("key directory");
|
||||
let backend = local_backend(&temp_dir).await;
|
||||
let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
config.allow_immediate_deletion = true;
|
||||
let manager = crate::manager::KmsManager::new(backend.clone(), config);
|
||||
let request = CreateKeyRequest {
|
||||
key_name: Some(PROBE_KEY_ID.to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(manager.create_key(request).await, Err(KmsError::InvalidOperation { .. })));
|
||||
let worker = worker(backend.clone());
|
||||
assert_eq!(worker.round().await, ProbeResult::Success, "the probe still provisions its reserved key");
|
||||
assert!(matches!(manager.disable_key(PROBE_KEY_ID).await, Err(KmsError::InvalidOperation { .. })));
|
||||
for immediate in [false, true] {
|
||||
assert!(matches!(
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: PROBE_KEY_ID.to_owned(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(immediate),
|
||||
confirm_key_id: Some(PROBE_KEY_ID.to_owned())
|
||||
})
|
||||
.await,
|
||||
Err(KmsError::InvalidOperation { .. })
|
||||
));
|
||||
assert_eq!(worker.round().await, ProbeResult::Success);
|
||||
}
|
||||
let customer = manager
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some("customer-probe-control".to_owned()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("customer key");
|
||||
manager
|
||||
.disable_key(&customer.key_id)
|
||||
.await
|
||||
.expect("customer key can be disabled");
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: customer.key_id,
|
||||
pending_window_in_days: Some(7),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("customer deletion remains available");
|
||||
assert_eq!(worker.round().await, ProbeResult::Success);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_backend_is_reported_unsupported_without_failures() {
|
||||
let (snapshot, result) = record_metrics(|| {
|
||||
|
||||
@@ -41,16 +41,6 @@ pub(super) fn pyroscope_log_filter() -> FilterFn {
|
||||
filter_fn(allows_non_pyroscope_dependency_event)
|
||||
}
|
||||
|
||||
/// s3s verbose events include response bodies and expected signatures. Keep
|
||||
/// diagnostics spans, but never let these events reach logs or OTLP exporters.
|
||||
pub(super) fn s3s_log_filter() -> FilterFn {
|
||||
filter_fn(|metadata| {
|
||||
!(metadata.is_event()
|
||||
&& (metadata.target() == "s3s" || metadata.target().starts_with("s3s::"))
|
||||
&& *metadata.level() > tracing::Level::INFO)
|
||||
})
|
||||
}
|
||||
|
||||
/// Build an `EnvFilter` from the given log level string.
|
||||
///
|
||||
/// If `default_level` is provided, it is used directly. Otherwise, the
|
||||
@@ -349,47 +339,6 @@ mod tests {
|
||||
format!("{HTTP_SERVER_LOG_TARGET}={level}")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3s_filter_blocks_credentials_despite_explicit_verbose_directives() {
|
||||
use tracing_subscriber::prelude::*;
|
||||
for directive in ["trace", concat!("trace,", "s3s::service=trace,", "s3s::ops::signature=debug")] {
|
||||
let writer = SharedWriter::default();
|
||||
let captured = Arc::clone(&writer.0);
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(EnvFilter::new(directive))
|
||||
.with(s3s_log_filter())
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_writer(writer)
|
||||
.with_span_events(tracing_subscriber::fmt::format::FmtSpan::NEW),
|
||||
);
|
||||
with_default(subscriber, || {
|
||||
let span = tracing::debug_span!(target: "s3s::service", "safe-operation-span");
|
||||
let _entered = span.enter();
|
||||
tracing::debug!(target: "s3s::service", response = "secret-response", "response dump");
|
||||
tracing::trace!(target: "s3s", response = "secret-trace", "response dump");
|
||||
tracing::debug!(target: "s3s::ops::signature", expected = "secret-signature", "signature mismatch");
|
||||
tracing::info!(target: "s3s::service", "visible-info");
|
||||
tracing::warn!(target: "s3s::service", "visible-warning");
|
||||
tracing::debug!(target: "rustfs", "visible-rustfs-debug");
|
||||
tracing::debug!(target: "s3s_other", "visible-other-target");
|
||||
});
|
||||
let output = String::from_utf8(captured.lock().expect("captured logs").clone()).expect("UTF-8 logs");
|
||||
assert!(!output.contains("secret"), "{output}");
|
||||
for expected in [
|
||||
"safe-operation-span",
|
||||
"visible-info",
|
||||
"visible-warning",
|
||||
"visible-rustfs-debug",
|
||||
"visible-other-target",
|
||||
] {
|
||||
assert!(output.contains(expected), "missing {expected}: {output}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_verbose_level() {
|
||||
assert!(is_verbose_level("debug"));
|
||||
|
||||
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
|
||||
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
|
||||
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3s_log_filter};
|
||||
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter};
|
||||
use crate::telemetry::rolling::{RollingAppender, Rotation};
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
@@ -369,7 +369,6 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(fmt_layer)
|
||||
.try_init()
|
||||
@@ -487,7 +486,6 @@ fn init_file_logging_internal(
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer)
|
||||
.with(stdout_layer)
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::set_observability_metric_enabled;
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
|
||||
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
|
||||
use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
|
||||
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
|
||||
@@ -296,7 +296,6 @@ pub(super) fn init_observability_http(
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer_opt)
|
||||
.with(stdout_layer_opt)
|
||||
@@ -1197,7 +1196,6 @@ mod tests {
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_ansi(false)
|
||||
@@ -1221,7 +1219,6 @@ mod tests {
|
||||
tracing_subscriber::registry()
|
||||
.with(build_env_filter("info", None))
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
|
||||
.try_init()
|
||||
.expect("install isolated LogTracer subscriber");
|
||||
|
||||
@@ -20,7 +20,7 @@ const ARN_PARTITION_RUSTFS: &str = "rustfs";
|
||||
const ARN_SERVICE_IAM: &str = "iam";
|
||||
const ARN_RESOURCE_TYPE_ROLE: &str = "role";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
pub struct ARN {
|
||||
pub partition: String,
|
||||
pub service: String,
|
||||
|
||||
@@ -165,10 +165,9 @@ impl<'de> Deserialize<'de> for ActionSet {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Hash, PartialEq, Eq, Clone, Debug, Copy)]
|
||||
#[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, Debug, Copy)]
|
||||
#[serde(try_from = "&str", untagged)]
|
||||
pub enum Action {
|
||||
All,
|
||||
S3Action(S3Action),
|
||||
AdminAction(AdminAction),
|
||||
StsAction(StsAction),
|
||||
@@ -176,29 +175,12 @@ pub enum Action {
|
||||
None,
|
||||
}
|
||||
|
||||
impl Serialize for Action {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Self::None => serializer.serialize_unit(),
|
||||
_ => serializer.serialize_str(self.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Action {
|
||||
pub fn is_match(&self, action: &Action) -> bool {
|
||||
self.is_match_for_effect(action, false)
|
||||
}
|
||||
|
||||
pub fn is_match_for_effect(&self, action: &Action, deny: bool) -> bool {
|
||||
if matches!(self, Self::All) {
|
||||
// Preserve the historical S3-only Allow while allowing deny-all and
|
||||
// NotAction exclusions to cover every action family.
|
||||
return deny || Self::S3Action(S3Action::AllActions).is_match_for_effect(action, false);
|
||||
}
|
||||
// Force-delete bypasses emptiness and version checks. A blanket `s3:*`
|
||||
// / `*` Allow (including canned consoleAdmin) must not confer it; the
|
||||
// statement has to name `s3:ForceDeleteBucket` or `s3:ForceDeleteObject`.
|
||||
@@ -220,7 +202,6 @@ fn action_requires_explicit_grant(action: &Action) -> bool {
|
||||
impl From<&Action> for &str {
|
||||
fn from(value: &Action) -> &'static str {
|
||||
match value {
|
||||
Action::All => "*",
|
||||
Action::S3Action(s) => s.into(),
|
||||
Action::AdminAction(s) => s.into(),
|
||||
Action::StsAction(s) => s.into(),
|
||||
@@ -240,8 +221,9 @@ impl Action {
|
||||
impl TryFrom<&str> for Action {
|
||||
type Error = Error;
|
||||
fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
|
||||
// Support wildcard "*" which matches all S3 actions (AWS S3 standard)
|
||||
if value == "*" {
|
||||
return Ok(Self::All);
|
||||
return Ok(Self::S3Action(S3Action::AllActions));
|
||||
}
|
||||
if value.starts_with(Self::S3_PREFIX) {
|
||||
Ok(Self::S3Action(
|
||||
@@ -840,7 +822,7 @@ mod tests {
|
||||
#[test]
|
||||
fn blanket_s3_wildcard_does_not_grant_force_delete() {
|
||||
let wildcard = Action::try_from("s3:*").expect("s3:* parses");
|
||||
let star = Action::try_from("*").expect("* parses");
|
||||
let star = Action::try_from("*").expect("* parses as s3:*");
|
||||
let force_bucket = Action::try_from("s3:ForceDeleteBucket").expect("force bucket action parses");
|
||||
let force_object = Action::try_from("s3:ForceDeleteObject").expect("force object action parses");
|
||||
let delete_object = Action::try_from("s3:DeleteObject").expect("delete object parses");
|
||||
@@ -859,10 +841,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_action_wildcard_parsing() {
|
||||
// Preserve the bare wildcard through serialization.
|
||||
// Test that "*" parses to S3Action::AllActions
|
||||
let action = Action::try_from("*").expect("Should parse wildcard");
|
||||
assert!(matches!(action, Action::All));
|
||||
assert_eq!(serde_json::to_string(&action).expect("serialize wildcard"), r#""*""#);
|
||||
assert!(matches!(action, Action::S3Action(S3Action::AllActions)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -948,13 +929,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_actionset_wildcard_serialization() {
|
||||
for wildcard in ["*", "s3:*"] {
|
||||
let actionset = ActionSet(vec![Action::try_from(wildcard).expect("parse wildcard")]);
|
||||
let json = serde_json::to_string(&actionset).expect("serialize action set");
|
||||
assert_eq!(json, format!(r#"["{wildcard}"]"#));
|
||||
let round_trip: ActionSet = serde_json::from_str(&json).expect("deserialize action set");
|
||||
assert_eq!(round_trip, actionset);
|
||||
}
|
||||
// Wildcard action should serialize as array for S3 specification compliance
|
||||
let actionset = ActionSet(vec![Action::try_from("*").expect("Should parse wildcard")]);
|
||||
|
||||
let json = serde_json::to_string(&actionset).expect("Should serialize");
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
|
||||
assert!(parsed.is_array(), "Should serialize as array");
|
||||
let arr = parsed.as_array().expect("Should be array");
|
||||
assert_eq!(arr.len(), 1);
|
||||
assert_eq!(arr[0].as_str().unwrap(), "s3:*");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use super::func::InnerFunc;
|
||||
use ipnetwork::IpNetwork;
|
||||
use serde::{Deserialize, Serialize, de::Visitor};
|
||||
use std::{collections::HashMap, net::IpAddr};
|
||||
use std::{borrow::Cow, collections::HashMap, net::IpAddr};
|
||||
|
||||
pub type AddrFunc = InnerFunc<AddrFuncValue>;
|
||||
|
||||
@@ -30,16 +30,7 @@ impl AddrFunc {
|
||||
};
|
||||
|
||||
for ip_net in inner.values.0.iter() {
|
||||
let canonical = ip.to_canonical();
|
||||
let matches_mapped_network = match (ip_net, canonical) {
|
||||
(IpNetwork::V6(network), IpAddr::V4(address))
|
||||
if network.prefix() >= 96 && network.ip().to_ipv4_mapped().is_some() =>
|
||||
{
|
||||
network.contains(address.to_ipv6_mapped())
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if ip_net.contains(ip) || ip_net.contains(canonical) || matches_mapped_network {
|
||||
if ip_net.contains(ip) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -90,7 +81,13 @@ impl<'de> Deserialize<'de> for AddrFuncValue {
|
||||
|
||||
impl AddrFuncValueVisitor {
|
||||
fn cidr<E: serde::de::Error>(v: &str) -> Result<IpNetwork, E> {
|
||||
v.parse::<IpNetwork>()
|
||||
let mut cidr_str = Cow::from(v);
|
||||
if v.find('/').is_none() {
|
||||
cidr_str.to_mut().push_str("/32");
|
||||
}
|
||||
|
||||
cidr_str
|
||||
.parse::<IpNetwork>()
|
||||
.map_err(|_| E::custom(format!("{v} can not be parsed to CIDR")))
|
||||
}
|
||||
}
|
||||
@@ -110,40 +107,6 @@ mod tests {
|
||||
};
|
||||
use test_case::test_case;
|
||||
|
||||
#[test]
|
||||
fn source_ip_matches_ipv4_and_mapped_forms_across_protocols() {
|
||||
for network in ["203.0.113.0/24", "::ffff:203.0.113.0/120"] {
|
||||
let function: AddrFunc =
|
||||
serde_json::from_str(&format!(r#"{{"aws:SourceIp":"{network}"}}"#)).expect("parse source IP condition");
|
||||
for address in ["203.0.113.9", "::ffff:203.0.113.9"] {
|
||||
assert!(
|
||||
function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![address.to_string()]),])),
|
||||
"{network} must match {address}"
|
||||
);
|
||||
}
|
||||
for address in ["198.51.100.9", "::ffff:198.51.100.9", "2001:db8::9"] {
|
||||
assert!(
|
||||
!function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![address.to_string()]),])),
|
||||
"{network} must not match {address}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_address_matches_only_the_named_host() {
|
||||
for (host, neighbor, prefix) in [
|
||||
("2001:db8:85a3::8a2e:370:7334", "2001:db8:ffff::1", 128),
|
||||
("203.0.113.7", "203.0.113.8", 32),
|
||||
] {
|
||||
let function: AddrFunc =
|
||||
serde_json::from_str(&format!(r#"{{"aws:SourceIp":"{host}"}}"#)).expect("parse bare IP address condition");
|
||||
assert_eq!(function.0[0].values.0[0].prefix(), prefix);
|
||||
assert!(function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![host.to_string()])])));
|
||||
assert!(!function.evaluate(&std::collections::HashMap::from([("SourceIp".to_string(), vec![neighbor.to_string()])])));
|
||||
}
|
||||
}
|
||||
|
||||
fn new_func(name: KeyName, variable: Option<String>, value: Vec<&str>) -> AddrFunc {
|
||||
AddrFunc {
|
||||
0: vec![FuncKeyValue {
|
||||
@@ -156,20 +119,20 @@ mod tests {
|
||||
#[test_case(r#"{"aws:SourceIp": "203.0.113.0/24"}"#, new_func(Aws(AWSSourceIP), None, vec!["203.0.113.0/24"]); "1")]
|
||||
#[test_case(r#"{"aws:SourceIp": "203.0.113.0"}"#, new_func(Aws(AWSSourceIP), None, vec!["203.0.113.0/32"]); "2")]
|
||||
#[test_case(r#"{"aws:SourceIp": "2001:DB8:1234:5678::/64"}"#, new_func(Aws(AWSSourceIP),None, vec!["2001:DB8:1234:5678::/64"]); "3")]
|
||||
#[test_case(r#"{"aws:SourceIp": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/128"]); "4")]
|
||||
#[test_case(r#"{"aws:SourceIp": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/32"]); "4")]
|
||||
#[test_case(r#"{"aws:SourceIp": ["203.0.113.0/24","203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), None, vec!["203.0.113.0/24", "203.0.113.0/32"]); "5")]
|
||||
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::/64","203.0.113.0/24"]}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/64", "203.0.113.0/24"]); "6")]
|
||||
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),None, vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/128"]); "7")]
|
||||
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/128", "203.0.113.0/32"]); "8")]
|
||||
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),None, vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/32"]); "7")]
|
||||
#[test_case(r#"{"aws:SourceIp": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), None, vec!["2001:DB8:1234:5678::/32", "203.0.113.0/32"]); "8")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": "203.0.113.0/24"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/24"]); "9")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": "203.0.113.0/24"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/24"]); "10")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": "203.0.113.0"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/32"]); "11")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": "2001:DB8:1234:5678::/64"}"#, new_func(Aws(AWSSourceIP),Some("a".into()), vec!["2001:DB8:1234:5678::/64"]); "12")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/128"]); "13")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": "2001:DB8:1234:5678::"}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/32"]); "13")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": ["203.0.113.0/24", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["203.0.113.0/24", "203.0.113.0/32"]); "14")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::/64", "203.0.113.0/24"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/64", "203.0.113.0/24"]); "15")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),Some("a".into()), vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/128"]); "16")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/128", "203.0.113.0/32"]); "17")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::"]}"#, new_func(Aws(AWSSourceIP),Some("a".into()), vec!["2001:DB8:1234:5678::/64", "2001:DB8:1234:5678::/32"]); "16")]
|
||||
#[test_case(r#"{"aws:SourceIp/a": ["2001:DB8:1234:5678::", "203.0.113.0"]}"#, new_func(Aws(AWSSourceIP), Some("a".into()), vec!["2001:DB8:1234:5678::/32", "203.0.113.0/32"]); "17")]
|
||||
fn test_deser(input: &str, expect: AddrFunc) -> Result<(), serde_json::Error> {
|
||||
let v: AddrFunc = serde_json::from_str(input)?;
|
||||
assert_eq!(v, expect);
|
||||
|
||||
@@ -61,11 +61,6 @@ pub enum Condition {
|
||||
|
||||
impl Condition {
|
||||
pub fn from_deserializer<'a, D: MapAccess<'a>>(key: &str, d: &mut D) -> Result<Self, D::Error> {
|
||||
// Two suffixes are retained for existing serialized policies. Bound the
|
||||
// chain before parsing so Clone, evaluation and Drop stay bounded too.
|
||||
if key.ends_with("IfExistsIfExistsIfExists") {
|
||||
return Err(Error::custom("condition operator has too many IfExists suffixes"));
|
||||
}
|
||||
Ok(match key {
|
||||
"StringEquals" => Self::StringEquals(d.next_value()?),
|
||||
"StringNotEquals" => Self::StringNotEquals(d.next_value()?),
|
||||
@@ -267,18 +262,18 @@ impl Condition {
|
||||
Null(s) => s.evaluate_null(values),
|
||||
Bool(s) => s.evaluate_bool(values),
|
||||
NumericEquals(s) => s.evaluate(i64::eq, false, values),
|
||||
NumericNotEquals(s) => s.evaluate(i64::ne, true, values),
|
||||
NumericNotEquals(s) => s.evaluate(i64::ne, false, values),
|
||||
NumericLessThan(s) => s.evaluate(i64::lt, false, values),
|
||||
NumericLessThanEquals(s) => s.evaluate(i64::le, false, values),
|
||||
NumericGreaterThan(s) => s.evaluate(i64::gt, false, values),
|
||||
NumericGreaterThanIfExists(s) => s.evaluate(i64::ge, true, values),
|
||||
NumericGreaterThanEquals(s) => s.evaluate(i64::ge, false, values),
|
||||
DateEquals(s) => s.evaluate(OffsetDateTime::eq, false, values),
|
||||
DateNotEquals(s) => s.evaluate(OffsetDateTime::ne, true, values),
|
||||
DateLessThan(s) => s.evaluate(OffsetDateTime::lt, false, values),
|
||||
DateLessThanEquals(s) => s.evaluate(OffsetDateTime::le, false, values),
|
||||
DateGreaterThan(s) => s.evaluate(OffsetDateTime::gt, false, values),
|
||||
DateGreaterThanEquals(s) => s.evaluate(OffsetDateTime::ge, false, values),
|
||||
DateEquals(s) => s.evaluate(OffsetDateTime::eq, values),
|
||||
DateNotEquals(s) => s.evaluate(OffsetDateTime::ne, values),
|
||||
DateLessThan(s) => s.evaluate(OffsetDateTime::lt, values),
|
||||
DateLessThanEquals(s) => s.evaluate(OffsetDateTime::le, values),
|
||||
DateGreaterThan(s) => s.evaluate(OffsetDateTime::gt, values),
|
||||
DateGreaterThanEquals(s) => s.evaluate(OffsetDateTime::ge, values),
|
||||
IfExists(inner) => {
|
||||
if !inner.has_any_key_in(values) {
|
||||
return true;
|
||||
@@ -491,31 +486,6 @@ mod tests {
|
||||
assert_eq!(functions, deserialized);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_if_exists_suffixes_are_rejected_before_recursion() {
|
||||
for count in [3, 10_000] {
|
||||
let key = format!("StringEquals{}", "IfExists".repeat(count));
|
||||
let document = serde_json::json!({key: {"s3:x-amz-server-side-encryption": "aws:kms"}});
|
||||
let result = serde_json::from_slice::<Functions>(&serde_json::to_vec(&document).expect("serialize condition"));
|
||||
assert!(result.is_err(), "{count} suffixes must be rejected without constructing a deep chain");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn numeric_and_date_not_equals_match_absent_keys() {
|
||||
for (operator, key, value, absent) in [
|
||||
("NumericNotEquals", "s3:max-keys", "30", true),
|
||||
("NumericEquals", "s3:max-keys", "30", false),
|
||||
("DateNotEquals", "s3:object-lock-retain-until-date", "2030-01-01T00:00:00Z", true),
|
||||
("DateEquals", "s3:object-lock-retain-until-date", "2030-01-01T00:00:00Z", false),
|
||||
] {
|
||||
let document = serde_json::json!({operator: {key: value}});
|
||||
let functions: Functions =
|
||||
serde_json::from_slice(&serde_json::to_vec(&document).expect("serialize condition")).expect("parse condition");
|
||||
assert_eq!(functions.evaluate(&HashMap::new()).await, absent, "{operator}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nested_if_exists_serde_round_trip() {
|
||||
// Verifies that nested IfExists(IfExists(StringEquals)) serializes with the
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::func::InnerFunc;
|
||||
use super::key_name::{AwsKeyName, KeyName};
|
||||
use serde::{Deserialize, Deserializer, Serialize, de};
|
||||
use std::{collections::HashMap, fmt};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
@@ -21,27 +20,14 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
pub type DateFunc = InnerFunc<DateFuncValue>;
|
||||
|
||||
impl DateFunc {
|
||||
pub fn evaluate(
|
||||
&self,
|
||||
op: impl Fn(&OffsetDateTime, &OffsetDateTime) -> bool,
|
||||
absent: bool,
|
||||
values: &HashMap<String, Vec<String>>,
|
||||
) -> bool {
|
||||
pub fn evaluate(&self, op: impl Fn(&OffsetDateTime, &OffsetDateTime) -> bool, values: &HashMap<String, Vec<String>>) -> bool {
|
||||
for inner in self.0.iter() {
|
||||
let v = match values.get(inner.key.name().as_str()).and_then(|x| x.first()) {
|
||||
Some(x) => x,
|
||||
None if absent => continue,
|
||||
None => return false,
|
||||
};
|
||||
|
||||
let parsed = if inner.key.is(&KeyName::Aws(AwsKeyName::AWSEpochTime)) {
|
||||
v.parse::<i64>()
|
||||
.ok()
|
||||
.and_then(|seconds| OffsetDateTime::from_unix_timestamp(seconds).ok())
|
||||
} else {
|
||||
OffsetDateTime::parse(v, &Rfc3339).ok()
|
||||
};
|
||||
let Some(rv) = parsed else {
|
||||
let Ok(rv) = OffsetDateTime::parse(v, &Rfc3339) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -102,7 +88,7 @@ impl<'de> Deserialize<'de> for DateFuncValue {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{AwsKeyName, DateFunc, DateFuncValue};
|
||||
use super::{DateFunc, DateFuncValue};
|
||||
use crate::policy::function::func::FuncKeyValue;
|
||||
use crate::policy::function::{
|
||||
key::Key,
|
||||
@@ -138,50 +124,15 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn date_conditions_compare_epoch_seconds_only_for_epoch_time() {
|
||||
let function = new_func(KeyName::Aws(AwsKeyName::AWSEpochTime), None, "2020-01-01T00:00:00Z");
|
||||
for (value, equal, later) in [
|
||||
("1577836799", false, false),
|
||||
("1577836800", true, false),
|
||||
("1577836801", false, true),
|
||||
] {
|
||||
let values = HashMap::from([("EpochTime".to_string(), vec![value.to_string()])]);
|
||||
assert_eq!(function.evaluate(OffsetDateTime::eq, false, &values), equal);
|
||||
assert_eq!(function.evaluate(OffsetDateTime::gt, false, &values), later);
|
||||
assert_eq!(function.evaluate(OffsetDateTime::lt, false, &values), !equal && !later);
|
||||
}
|
||||
for invalid in ["2020-01-01T00:00:00Z", "invalid", "9223372036854775807"] {
|
||||
let values = HashMap::from([("EpochTime".to_string(), vec![invalid.to_string()])]);
|
||||
assert!(!function.evaluate(OffsetDateTime::ne, true, &values));
|
||||
}
|
||||
let header_date = new_func(S3(S3ObjectLockRetainUntilDate), None, "2020-01-01T00:00:00Z");
|
||||
let values = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["1577836800".to_string()])]);
|
||||
assert!(!header_date.evaluate(OffsetDateTime::eq, false, &values));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_date_key_does_not_skip_other_keys() {
|
||||
let mut function = new_func(S3(S3ObjectLockRetainUntilDate), None, "2030-01-01T00:00:00Z");
|
||||
function
|
||||
.0
|
||||
.extend(new_func(KeyName::Aws(AwsKeyName::AWSCurrentTime), None, "2030-01-01T00:00:00Z").0);
|
||||
let values = HashMap::from([("CurrentTime".to_string(), vec!["2030-01-01T00:00:00Z".to_string()])]);
|
||||
assert!(!function.evaluate(OffsetDateTime::ne, true, &values));
|
||||
assert!(function.evaluate(OffsetDateTime::eq, true, &values));
|
||||
function.0.reverse();
|
||||
assert!(!function.evaluate(OffsetDateTime::ne, true, &values));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evaluate_compares_request_date_to_policy_date() {
|
||||
let function = new_func(S3(S3ObjectLockRetainUntilDate), None, "2030-01-01T00:00:00Z");
|
||||
let later = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2099-01-01T00:00:00Z".to_string()])]);
|
||||
let earlier = HashMap::from([("object-lock-retain-until-date".to_string(), vec!["2029-01-01T00:00:00Z".to_string()])]);
|
||||
|
||||
assert!(function.evaluate(OffsetDateTime::gt, false, &later));
|
||||
assert!(!function.evaluate(OffsetDateTime::gt, false, &earlier));
|
||||
assert!(function.evaluate(OffsetDateTime::lt, false, &earlier));
|
||||
assert!(!function.evaluate(OffsetDateTime::lt, false, &later));
|
||||
assert!(function.evaluate(OffsetDateTime::gt, &later));
|
||||
assert!(!function.evaluate(OffsetDateTime::gt, &earlier));
|
||||
assert!(function.evaluate(OffsetDateTime::lt, &earlier));
|
||||
assert!(!function.evaluate(OffsetDateTime::lt, &later));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,12 +26,11 @@ pub type NumberFunc = InnerFunc<NumberFuncValue>;
|
||||
pub struct NumberFuncValue(i64);
|
||||
|
||||
impl NumberFunc {
|
||||
pub fn evaluate(&self, op: impl Fn(&i64, &i64) -> bool, absent: bool, values: &HashMap<String, Vec<String>>) -> bool {
|
||||
pub fn evaluate(&self, op: impl Fn(&i64, &i64) -> bool, if_exists: bool, values: &HashMap<String, Vec<String>>) -> bool {
|
||||
for inner in self.0.iter() {
|
||||
let v = match values.get(inner.key.name().as_str()).and_then(|x| x.first()) {
|
||||
Some(x) => x,
|
||||
None if absent => continue,
|
||||
None => return false,
|
||||
None => return if_exists,
|
||||
};
|
||||
|
||||
let Ok(rv) = v.parse::<i64>() else {
|
||||
@@ -147,20 +146,6 @@ mod tests {
|
||||
assert_eq!(limit.evaluate(i64::eq, false, &values), equals);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_numeric_key_does_not_skip_other_keys() {
|
||||
let mut function = new_func(S3(S3MaxKeys), None, 30);
|
||||
function
|
||||
.0
|
||||
.extend(new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30).0);
|
||||
let values = HashMap::from([("object-lock-remaining-retention-days".to_string(), vec!["30".to_string()])]);
|
||||
assert!(!function.evaluate(i64::ne, true, &values));
|
||||
assert!(!function.evaluate(i64::gt, true, &values));
|
||||
assert!(function.evaluate(i64::eq, true, &values));
|
||||
function.0.reverse();
|
||||
assert!(!function.evaluate(i64::ne, true, &values));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remaining_retention_days_numeric_condition_without_value_uses_if_exists() {
|
||||
let limit = new_func(S3(S3ObjectLockRemainingRetentionDays), None, 30);
|
||||
|
||||
@@ -3416,277 +3416,6 @@ mod test {
|
||||
assert_eq!(round_trip.statements[0].effect, policy.statements[0].effect);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn global_wildcard_deny_covers_all_families_without_broadening_allow() {
|
||||
for action in [
|
||||
"s3:GetObject",
|
||||
"s3:ForceDeleteObject",
|
||||
"admin:CreateUser",
|
||||
"sts:AssumeRole",
|
||||
"kms:Configure",
|
||||
"kms:Decrypt",
|
||||
] {
|
||||
let parsed_action = Action::try_from(action).expect("action parses");
|
||||
let conditions = HashMap::new();
|
||||
let claims = HashMap::new();
|
||||
let args = Args {
|
||||
account: "user",
|
||||
groups: &None,
|
||||
action: parsed_action,
|
||||
bucket: if action.starts_with("s3:") { "bucket" } else { "" },
|
||||
object: if action.starts_with("s3:") { "object" } else { "" },
|
||||
conditions: &conditions,
|
||||
claims: &claims,
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
};
|
||||
for (effect, selector, wildcard, expected) in [
|
||||
("Deny", "Action", "*", false),
|
||||
("Deny", "Action", "s3:*", !action.starts_with("s3:")),
|
||||
("Deny", "NotAction", "*", true),
|
||||
] {
|
||||
let document = serde_json::json!({
|
||||
"Version": "2012-10-17", "Statement": [
|
||||
{"Effect": "Allow", "Action": action, "Resource": "*"},
|
||||
{"Effect": effect, selector: wildcard, "Resource": "*"}
|
||||
]
|
||||
});
|
||||
let policy = Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy"))
|
||||
.expect("parse wildcard policy");
|
||||
assert_eq!(policy.is_allowed(&args).await, expected, "{effect} {selector} {wildcard} for {action}");
|
||||
let serialized = serde_json::to_value(&policy).expect("serialize policy");
|
||||
assert_eq!(serialized["Statement"][1][selector][0], wildcard);
|
||||
}
|
||||
for selector in ["Action", "NotAction"] {
|
||||
let document = serde_json::json!({
|
||||
"Version": "2012-10-17", "Statement": [{"Effect": "Allow", selector: "*", "Resource": "*"}]
|
||||
});
|
||||
let policy = Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy"))
|
||||
.expect("parse wildcard Allow");
|
||||
assert_eq!(policy.is_allowed(&args).await, selector == "Action" && action == "s3:GetObject");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn global_wildcard_deny_still_obeys_resource_and_condition_scope() {
|
||||
let document = serde_json::json!({
|
||||
"Version": "2012-10-17", "Statement": [
|
||||
{"Effect": "Allow", "Action": "admin:CreateUser", "Resource": "*"},
|
||||
{"Effect": "Deny", "Action": "*", "Resource": "arn:aws:s3:::bucket/*"}
|
||||
]
|
||||
});
|
||||
let policy = Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy")).expect("parse scoped Deny");
|
||||
let claims = HashMap::new();
|
||||
let conditions = HashMap::new();
|
||||
let mut args = Args {
|
||||
account: "user",
|
||||
groups: &None,
|
||||
action: Action::try_from("admin:CreateUser").expect("admin action"),
|
||||
bucket: "",
|
||||
object: "",
|
||||
conditions: &conditions,
|
||||
claims: &claims,
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
};
|
||||
assert!(policy.is_allowed(&args).await, "bucket-scoped Deny must not become a global admin Deny");
|
||||
let mut document = document;
|
||||
document["Statement"][1]["Resource"] = serde_json::json!("*");
|
||||
document["Statement"][1]["Condition"] = serde_json::json!({"StringEquals": {"aws:username": "blocked"}});
|
||||
let conditional =
|
||||
Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy")).expect("parse conditional Deny");
|
||||
assert!(conditional.is_allowed(&args).await);
|
||||
let conditions = HashMap::from([("username".to_string(), vec!["blocked".to_string()])]);
|
||||
args.conditions = &conditions;
|
||||
assert!(!conditional.is_allowed(&args).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn security_conditions_enforce_identity_and_bucket_denies() {
|
||||
for (operator, key, policy_value, request_value, denied) in [
|
||||
("NumericNotEquals", "s3:object-lock-remaining-retention-days", "30", None, true),
|
||||
("NumericNotEquals", "s3:object-lock-remaining-retention-days", "30", Some("30"), false),
|
||||
("NumericNotEquals", "s3:object-lock-remaining-retention-days", "30", Some("10"), true),
|
||||
("DateNotEquals", "s3:object-lock-retain-until-date", "2030-01-01T00:00:00Z", None, true),
|
||||
(
|
||||
"DateNotEquals",
|
||||
"s3:object-lock-retain-until-date",
|
||||
"2030-01-01T00:00:00Z",
|
||||
Some("2030-01-01T00:00:00Z"),
|
||||
false,
|
||||
),
|
||||
("DateGreaterThan", "aws:EpochTime", "2020-01-01T00:00:00Z", Some("1577836801"), true),
|
||||
("DateGreaterThan", "aws:EpochTime", "2020-01-01T00:00:00Z", Some("1577836799"), false),
|
||||
(
|
||||
"NotIpAddress",
|
||||
"aws:SourceIp",
|
||||
"2001:db8:85a3::8a2e:370:7334",
|
||||
Some("2001:db8:ffff::1"),
|
||||
true,
|
||||
),
|
||||
(
|
||||
"NotIpAddress",
|
||||
"aws:SourceIp",
|
||||
"2001:db8:85a3::8a2e:370:7334",
|
||||
Some("2001:db8:85a3::8a2e:370:7334"),
|
||||
false,
|
||||
),
|
||||
(
|
||||
"IpAddress",
|
||||
"aws:SourceIp",
|
||||
"2001:db8:85a3::8a2e:370:7334",
|
||||
Some("2001:db8:ffff::1"),
|
||||
false,
|
||||
),
|
||||
("IpAddress", "aws:SourceIp", "2001:db8::/32", Some("2001:db8:ffff::1"), true),
|
||||
] {
|
||||
let mut document = serde_json::json!({
|
||||
"Version": "2012-10-17", "Statement": [
|
||||
{"Effect": "Allow", "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*"},
|
||||
{"Effect": "Deny", "Action": "s3:PutObject", "Resource": "arn:aws:s3:::bucket/*",
|
||||
"Condition": {operator: {key: policy_value}}}
|
||||
]
|
||||
});
|
||||
let policy =
|
||||
Policy::parse_config(&serde_json::to_vec(&document).expect("serialize policy")).expect("parse identity policy");
|
||||
let conditions = request_value
|
||||
.map(|value| {
|
||||
HashMap::from([(key.split_once(':').expect("condition prefix").1.to_string(), vec![value.to_string()])])
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let claims = HashMap::new();
|
||||
let args = Args {
|
||||
account: "user",
|
||||
groups: &None,
|
||||
action: Action::S3Action(S3Action::PutObjectAction),
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
conditions: &conditions,
|
||||
claims: &claims,
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
};
|
||||
assert_eq!(policy.is_allowed(&args).await, !denied, "identity {operator} {key} {request_value:?}");
|
||||
for statement in document["Statement"].as_array_mut().expect("statements array") {
|
||||
statement["Principal"] = serde_json::json!("*");
|
||||
}
|
||||
let policy: BucketPolicy = serde_json::from_slice(&serde_json::to_vec(&document).expect("serialize bucket policy"))
|
||||
.expect("parse bucket policy");
|
||||
assert_eq!(
|
||||
policy
|
||||
.is_allowed(&BucketPolicyArgs {
|
||||
account: "user",
|
||||
groups: &None,
|
||||
action: args.action,
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
})
|
||||
.await,
|
||||
!denied,
|
||||
"bucket {operator} {key} {request_value:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn service_principal_never_grants_access_to_a_named_access_key() {
|
||||
for principal in [
|
||||
serde_json::json!({"Service": "logging.s3.amazonaws.com"}),
|
||||
serde_json::json!({"Service": "*"}),
|
||||
serde_json::json!({"Service": "logging.s3.amazonaws.com", "AWS": "authorized-user"}),
|
||||
] {
|
||||
let document = serde_json::json!({
|
||||
"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": principal,
|
||||
"Action": ["s3:GetObject", "s3:PutObject"], "Resource": "arn:aws:s3:::bucket/*"}]
|
||||
});
|
||||
let policy: BucketPolicy = serde_json::from_slice(&serde_json::to_vec(&document).unwrap()).unwrap();
|
||||
for action in [S3Action::GetObjectAction, S3Action::PutObjectAction] {
|
||||
for account in ["logging.s3.amazonaws.com", "authorized-user"] {
|
||||
let allowed = policy
|
||||
.is_allowed(&BucketPolicyArgs {
|
||||
account,
|
||||
groups: &None,
|
||||
action: Action::S3Action(action),
|
||||
bucket: "bucket",
|
||||
object: "log",
|
||||
conditions: &HashMap::new(),
|
||||
is_owner: false,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(allowed, account == "authorized-user" && principal.get("AWS").is_some());
|
||||
}
|
||||
}
|
||||
assert_eq!(serde_json::to_value(&policy).unwrap()["Statement"][0]["Principal"], principal);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_objects_obey_prefix_denies_in_identity_and_bucket_policies() {
|
||||
let mut document = serde_json::json!({
|
||||
"Version": "2012-10-17", "Statement": [
|
||||
{"Effect": "Allow", "Action": "s3:*", "Resource": "arn:aws:s3:::bucket/*"},
|
||||
{"Effect": "Deny", "Action": "s3:*", "Resource": "arn:aws:s3:::bucket/private/*"}
|
||||
]
|
||||
});
|
||||
let identity = Policy::parse_config(&serde_json::to_vec(&document).unwrap()).unwrap();
|
||||
for statement in document["Statement"].as_array_mut().unwrap() {
|
||||
statement["Principal"] = serde_json::json!("*");
|
||||
}
|
||||
let bucket: BucketPolicy = serde_json::from_slice(&serde_json::to_vec(&document).unwrap()).unwrap();
|
||||
for action in [
|
||||
S3Action::GetObjectAction,
|
||||
S3Action::PutObjectAction,
|
||||
S3Action::DeleteObjectAction,
|
||||
S3Action::DeleteObjectVersionAction,
|
||||
] {
|
||||
for (object, allowed) in [
|
||||
("private/", false),
|
||||
("private/child", false),
|
||||
("public/", true),
|
||||
("private", true),
|
||||
] {
|
||||
let conditions = HashMap::new();
|
||||
assert_eq!(
|
||||
identity
|
||||
.is_allowed(&Args {
|
||||
account: "user",
|
||||
groups: &None,
|
||||
action: Action::S3Action(action),
|
||||
bucket: "bucket",
|
||||
object,
|
||||
conditions: &conditions,
|
||||
claims: &HashMap::new(),
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
})
|
||||
.await,
|
||||
allowed,
|
||||
"identity {action:?} {object}"
|
||||
);
|
||||
for account in ["user", ""] {
|
||||
assert_eq!(
|
||||
bucket
|
||||
.is_allowed(&BucketPolicyArgs {
|
||||
account,
|
||||
groups: &None,
|
||||
action: Action::S3Action(action),
|
||||
bucket: "bucket",
|
||||
object,
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
})
|
||||
.await,
|
||||
allowed,
|
||||
"bucket {account} {action:?} {object}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_delete_actions_require_an_explicit_grant() {
|
||||
let wildcard = Policy::parse_config(
|
||||
|
||||
@@ -19,7 +19,7 @@ use std::collections::HashSet;
|
||||
|
||||
/// Principal that serializes AWS field as single string when containing only "*",
|
||||
/// or as an array otherwise (matching AWS S3 API format).
|
||||
/// Service principals round-trip for compatibility but do not match access keys.
|
||||
/// Also supports Service principals (e.g., "logging.s3.amazonaws.com").
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Principal {
|
||||
aws: HashSet<String>,
|
||||
@@ -131,6 +131,11 @@ impl Principal {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for pattern in self.service.iter() {
|
||||
if wildcard::is_simple_match(pattern, principal) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -220,7 +225,7 @@ mod test {
|
||||
service: HashSet::from(["logging.s3.amazonaws.com".to_string()]),
|
||||
};
|
||||
|
||||
assert!(!principal.is_match("logging.s3.amazonaws.com"));
|
||||
assert!(principal.is_match("logging.s3.amazonaws.com"));
|
||||
assert!(!principal.is_match("replication.s3.amazonaws.com"));
|
||||
assert!(!principal.is_match("arn:aws:iam::123456789012:root"));
|
||||
}
|
||||
@@ -233,7 +238,7 @@ mod test {
|
||||
};
|
||||
|
||||
assert!(principal.is_match("arn:aws:iam::123456789012:root"));
|
||||
assert!(!principal.is_match("logging.s3.amazonaws.com"));
|
||||
assert!(principal.is_match("logging.s3.amazonaws.com"));
|
||||
assert!(!principal.is_match("other-principal"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,11 +17,7 @@ use serde::{
|
||||
Deserialize, Deserializer, Serialize,
|
||||
de::{self, Error as DeError, Visitor},
|
||||
};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
fmt,
|
||||
ops::Deref,
|
||||
};
|
||||
use std::{collections::HashMap, fmt, ops::Deref};
|
||||
|
||||
use super::{
|
||||
Error as IamError, Validator,
|
||||
@@ -54,6 +50,12 @@ impl ResourceSet {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
fn push_unique(&mut self, resource: Resource) {
|
||||
if !self.0.contains(&resource) {
|
||||
self.0.push(resource);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn is_match(&self, resource: &str, conditions: &HashMap<String, Vec<String>>) -> bool {
|
||||
self.is_match_with_resolver(resource, conditions, None).await
|
||||
}
|
||||
@@ -128,7 +130,9 @@ impl<'de> Deserialize<'de> for ResourceSet {
|
||||
E: de::Error,
|
||||
{
|
||||
let resource = Resource::try_from(value).map_err(|e| E::custom(format!("invalid resource: {}", e)))?;
|
||||
Ok(ResourceSet(vec![resource]))
|
||||
let mut resources = ResourceSet::default();
|
||||
resources.push_unique(resource);
|
||||
Ok(resources)
|
||||
}
|
||||
|
||||
fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
|
||||
@@ -136,14 +140,11 @@ impl<'de> Deserialize<'de> for ResourceSet {
|
||||
A: de::SeqAccess<'de>,
|
||||
A::Error: DeError,
|
||||
{
|
||||
let mut resources = ResourceSet::default();
|
||||
let mut seen = HashSet::new();
|
||||
let mut resources = ResourceSet(Vec::with_capacity(seq.size_hint().unwrap_or(0)));
|
||||
while let Some(value) = seq.next_element::<String>()? {
|
||||
match Resource::try_from(value.as_str()) {
|
||||
Ok(resource) => {
|
||||
if seen.insert(resource.clone()) {
|
||||
resources.0.push(resource);
|
||||
}
|
||||
resources.push_unique(resource);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(A::Error::custom(format!("invalid resource: {}", e)));
|
||||
@@ -223,14 +224,7 @@ impl Resource {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Keep directory objects distinct from their bare key for prefix
|
||||
// matching, while retaining path cleaning and exact-match compatibility.
|
||||
let wildcard_resource = if matches!(self, Resource::S3(_)) && resource.ends_with('/') && cp.contains('/') {
|
||||
format!("{cp}/")
|
||||
} else {
|
||||
cp.clone()
|
||||
};
|
||||
if cp != "." && wildcard::is_match(resolved_pattern, &wildcard_resource) {
|
||||
if cp != "." && wildcard::is_match(resolved_pattern, &cp) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -320,20 +314,6 @@ mod tests {
|
||||
use std::collections::HashMap;
|
||||
use test_case::test_case;
|
||||
|
||||
#[test]
|
||||
fn large_resource_sets_deduplicate_without_changing_first_seen_order() {
|
||||
let unique = (0..40_000)
|
||||
.map(|index| format!("arn:aws:s3:::bucket/key-{index}"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut repeated = unique.clone();
|
||||
repeated.extend(unique.iter().rev().cloned());
|
||||
let resources: super::ResourceSet = serde_json::from_slice(&serde_json::to_vec(&repeated).expect("encode resources"))
|
||||
.expect("large valid resource list parses");
|
||||
assert_eq!(resources.len(), unique.len());
|
||||
assert_eq!(serde_json::to_value(resources).expect("serialize resources"), serde_json::json!(unique));
|
||||
assert!(serde_json::from_str::<super::ResourceSet>(r#"["arn:aws:s3:::bucket/key","invalid"]"#).is_err());
|
||||
}
|
||||
|
||||
#[test_case("*", "mybucket" => true; "all buckets")]
|
||||
#[test_case("*", "mybucket/myobject" => true; "all objects")]
|
||||
#[test_case("*", "key/mykey" => true; "all keys")]
|
||||
@@ -355,10 +335,6 @@ mod tests {
|
||||
#[test_case("arn:aws:s3:::mybucket","mybucket/myobject" => false; "15")]
|
||||
#[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/evil.txt" => false; "16")]
|
||||
#[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/safe/../../victim-bucket/evil.txt" => false; "17")]
|
||||
#[test_case("arn:aws:s3:::bucket/secret/*","bucket/secret/" => true; "directory prefix")]
|
||||
#[test_case("arn:aws:s3:::bucket/secret","bucket/secret/" => true; "directory exact compatibility")]
|
||||
#[test_case("arn:aws:s3:::bucket/*","bucket/" => false; "bucket is not a directory object")]
|
||||
#[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/" => false; "directory traversal")]
|
||||
#[test_case("arn:aws:kms:::key/mykey","key/mykey" => true; "kms exact key")]
|
||||
#[test_case("arn:aws:kms:::key/mykey","key/otherkey" => false; "kms wrong key")]
|
||||
#[test_case("arn:aws:kms:::key/*","key/mykey" => true; "kms key wildcard")]
|
||||
|
||||
@@ -144,7 +144,7 @@ impl Statement {
|
||||
|
||||
for action in self.actions.iter() {
|
||||
match action {
|
||||
Action::All | Action::S3Action(_) => saw_s3 = true,
|
||||
Action::S3Action(_) => saw_s3 = true,
|
||||
Action::AdminAction(_) => saw_admin = true,
|
||||
Action::StsAction(_) => saw_sts = true,
|
||||
Action::KmsAction(_) => saw_kms = true,
|
||||
|
||||
@@ -259,11 +259,35 @@ async fn resolve_aws_variables_with_depth(pattern: &str, resolver: &dyn PolicyVa
|
||||
if depth >= 10 {
|
||||
return vec![pattern.to_string()];
|
||||
}
|
||||
// Values from credentials and claims are data, not policy source. Expanding
|
||||
// substituted placeholders again allows attacker-controlled exponential growth.
|
||||
let mut results = resolve_single_pass(pattern, resolver, depth).await;
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
results.retain(|result| seen.insert(result.clone()));
|
||||
let mut results = vec![pattern.to_string()];
|
||||
|
||||
let mut changed = true;
|
||||
let max_iterations = 10; // Prevent infinite loops
|
||||
let mut iteration = 0;
|
||||
|
||||
while changed && iteration < max_iterations {
|
||||
changed = false;
|
||||
iteration += 1;
|
||||
|
||||
let mut new_results = Vec::new();
|
||||
for result in &results {
|
||||
let resolved = resolve_single_pass(result, resolver, depth).await;
|
||||
if resolved.len() > 1 || (resolved.len() == 1 && &resolved[0] != result) {
|
||||
changed = true;
|
||||
}
|
||||
new_results.extend(resolved);
|
||||
}
|
||||
|
||||
// Remove duplicates while preserving order
|
||||
results.clear();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for result in new_results {
|
||||
if seen.insert(result.clone()) {
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
@@ -346,8 +370,8 @@ async fn resolve_single_pass(pattern: &str, resolver: &dyn PolicyVariableResolve
|
||||
let suffix = &results[i].0[end_pos + 1..];
|
||||
|
||||
for value in values {
|
||||
// Resume after the substitution; generated placeholders
|
||||
// are literal data and must never be expanded.
|
||||
// Resume after the substitution; a generated placeholder
|
||||
// belongs to the next bounded pass, not this one.
|
||||
new_results.push((format!("{prefix}{value}{suffix}"), actual_pos + value.len()));
|
||||
}
|
||||
|
||||
@@ -421,20 +445,8 @@ mod tests {
|
||||
calls: AtomicUsize::new(0),
|
||||
};
|
||||
let result = resolve_aws_variables("${aws:AccountId}{aws:username}", &resolver).await;
|
||||
assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(result, vec!["$${aws:username}".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn substituted_account_id_placeholders_are_literal() {
|
||||
let account = "${aws:AccountId}".repeat(2);
|
||||
let resolver = VariableResolver::new(VariableContext {
|
||||
account_id: Some(account.clone()),
|
||||
username: Some("alice".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
let result = resolve_aws_variables("prefix/${aws:AccountId}/${aws:username}", &resolver).await;
|
||||
assert_eq!(result, vec![format!("prefix/{account}/alice")]);
|
||||
assert!(resolver.calls.load(Ordering::Relaxed) <= 10);
|
||||
assert_eq!(result, vec!["${aws:AccountId}{aws:username}".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -30,7 +30,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-tls-runtime?/fips"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
@@ -183,7 +182,6 @@ subtle = { workspace = true, optional = true }
|
||||
socket2 = { workspace = true, optional = true, features = ["all"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rcgen = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
proptest = "1"
|
||||
rustfs-test-utils = { workspace = true }
|
||||
|
||||
@@ -16,9 +16,6 @@ pub mod client;
|
||||
pub mod gateway;
|
||||
pub mod session;
|
||||
|
||||
#[cfg(any(feature = "webdav", feature = "ftps"))]
|
||||
pub(crate) mod tls;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod dummy_storage;
|
||||
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use rustls::{ServerConfig, server::ResolvesServerCert};
|
||||
use std::{io, path::Path, sync::Arc};
|
||||
|
||||
/// Shared listener TLS policy: a configured client CA always requires client authentication.
|
||||
pub(crate) fn server_config(resolver: Arc<dyn ResolvesServerCert>, ca_file: Option<&str>) -> io::Result<ServerConfig> {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
let builder = ServerConfig::builder();
|
||||
let builder = if let Some(ca_file) = ca_file {
|
||||
builder.with_client_cert_verifier(rustfs_tls_runtime::build_webpki_client_verifier_from_ca_file(Path::new(ca_file))?)
|
||||
} else {
|
||||
builder.with_no_client_auth()
|
||||
};
|
||||
Ok(builder.with_cert_resolver(resolver))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose};
|
||||
use rustls::{ClientConfig, ClientConnection, RootCertStore, ServerConnection, pki_types::PrivatePkcs8KeyDer};
|
||||
|
||||
fn authority(name: &str) -> CertifiedIssuer<'static, KeyPair> {
|
||||
let mut params = CertificateParams::new(vec![name.to_owned()]).expect("CA names");
|
||||
params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
|
||||
CertifiedIssuer::self_signed(params, KeyPair::generate().expect("CA signing key")).expect("self-signed CA")
|
||||
}
|
||||
|
||||
fn handshake(server: ServerConfig, client: ClientConfig) -> io::Result<()> {
|
||||
let mut server = ServerConnection::new(Arc::new(server)).map_err(io::Error::other)?;
|
||||
let mut client =
|
||||
ClientConnection::new(Arc::new(client), "localhost".try_into().expect("server name")).map_err(io::Error::other)?;
|
||||
for _ in 0..16 {
|
||||
let mut wire = Vec::new();
|
||||
client.write_tls(&mut wire)?;
|
||||
if !wire.is_empty() {
|
||||
server.read_tls(&mut std::io::Cursor::new(wire))?;
|
||||
server.process_new_packets().map_err(io::Error::other)?;
|
||||
}
|
||||
let mut wire = Vec::new();
|
||||
server.write_tls(&mut wire)?;
|
||||
if !wire.is_empty() {
|
||||
client.read_tls(&mut std::io::Cursor::new(wire))?;
|
||||
client.process_new_packets().map_err(io::Error::other)?;
|
||||
}
|
||||
if !server.is_handshaking() && !client.is_handshaking() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(io::Error::other("TLS handshake did not finish"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_listener_tls_requires_the_configured_client_ca() {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
let directory = tempfile::tempdir().expect("TLS directory");
|
||||
let trusted = authority("trusted-ca");
|
||||
let unrelated = authority("unrelated-ca");
|
||||
let server_key = KeyPair::generate().expect("server key");
|
||||
let mut params = CertificateParams::new(vec!["localhost".to_owned()]).expect("server names");
|
||||
params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
|
||||
let server_cert = params.signed_by(&server_key, &trusted).expect("server certificate");
|
||||
std::fs::write(directory.path().join("rustfs_cert.pem"), server_cert.pem()).expect("write server certificate");
|
||||
std::fs::write(directory.path().join("rustfs_key.pem"), server_key.serialize_pem()).expect("write server key");
|
||||
let ca_file = directory.path().join("clients.pem");
|
||||
std::fs::write(&ca_file, trusted.pem()).expect("write client CA");
|
||||
let resolver =
|
||||
rustfs_tls_runtime::ReloadableServerCertResolver::load_from_directory(directory.path().to_str().expect("directory"))
|
||||
.expect("production certificate resolver");
|
||||
for (client_issuer, allowed) in [(None, false), (Some(&unrelated), false), (Some(&trusted), true)] {
|
||||
let mut roots = RootCertStore::empty();
|
||||
roots.add(trusted.der().clone()).expect("trust server CA");
|
||||
let builder = ClientConfig::builder().with_root_certificates(roots);
|
||||
let client = if let Some(issuer) = client_issuer {
|
||||
let key = KeyPair::generate().expect("client key");
|
||||
let mut params = CertificateParams::new(Vec::new()).expect("client names");
|
||||
params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
|
||||
let cert = params.signed_by(&key, issuer).expect("client certificate");
|
||||
builder
|
||||
.with_client_auth_cert(vec![cert.der().clone()], PrivatePkcs8KeyDer::from(key.serialize_der()).into())
|
||||
.expect("client identity")
|
||||
} else {
|
||||
builder.with_no_client_auth()
|
||||
};
|
||||
let server = server_config(resolver.clone(), ca_file.to_str()).expect("production listener TLS policy");
|
||||
assert_eq!(handshake(server, client).is_ok(), allowed);
|
||||
}
|
||||
let mut roots = RootCertStore::empty();
|
||||
roots.add(trusted.der().clone()).expect("trust server CA");
|
||||
handshake(
|
||||
server_config(resolver.clone(), None).expect("ordinary TLS"),
|
||||
ClientConfig::builder().with_root_certificates(roots).with_no_client_auth(),
|
||||
)
|
||||
.expect("no CA preserves ordinary TLS");
|
||||
for bad in [
|
||||
"",
|
||||
"not a certificate",
|
||||
"-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----",
|
||||
] {
|
||||
std::fs::write(&ca_file, bad).expect("bad CA fixture");
|
||||
assert!(server_config(resolver.clone(), ca_file.to_str()).is_err(), "bad CA must fail startup");
|
||||
}
|
||||
std::fs::remove_file(&ca_file).expect("remove CA fixture");
|
||||
assert!(server_config(resolver, ca_file.to_str()).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn protocol_listeners_refuse_startup_with_an_invalid_client_ca() {
|
||||
let directory = tempfile::tempdir().expect("TLS directory");
|
||||
let certified = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).expect("server identity");
|
||||
std::fs::write(directory.path().join("rustfs_cert.pem"), certified.cert.pem()).expect("server certificate");
|
||||
std::fs::write(directory.path().join("rustfs_key.pem"), certified.signing_key.serialize_pem()).expect("server key");
|
||||
let ca_file = directory.path().join("clients.pem");
|
||||
std::fs::write(&ca_file, "not a CA certificate").expect("invalid client CA fixture");
|
||||
let (_shutdown, shutdown) = tokio::sync::broadcast::channel(1);
|
||||
#[cfg(feature = "webdav")]
|
||||
{
|
||||
let config = crate::webdav::config::WebDavConfig {
|
||||
bind_addr: "127.0.0.1:0".parse().expect("bind"),
|
||||
tls_enabled: true,
|
||||
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
|
||||
ca_file: Some(ca_file.to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let server = crate::webdav::server::WebDavServer::new(config, crate::common::dummy_storage::DummyBackend::default())
|
||||
.await
|
||||
.expect("path checks");
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), server.start(shutdown.resubscribe()))
|
||||
.await
|
||||
.expect("startup deadline")
|
||||
.expect_err("invalid CA must not serve WebDAV");
|
||||
assert!(matches!(error, crate::webdav::config::WebDavInitError::Tls(_)));
|
||||
}
|
||||
#[cfg(feature = "ftps")]
|
||||
{
|
||||
let config = crate::ftps::config::FtpsConfig {
|
||||
bind_addr: "127.0.0.1:0".parse().expect("bind"),
|
||||
tls_enabled: true,
|
||||
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
|
||||
ca_file: Some(ca_file.to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let server = crate::ftps::server::FtpsServer::new(config, crate::common::dummy_storage::DummyBackend::default())
|
||||
.await
|
||||
.expect("path checks");
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), server.start(shutdown.resubscribe()))
|
||||
.await
|
||||
.expect("startup deadline")
|
||||
.expect_err("invalid CA must not serve FTPS");
|
||||
assert!(matches!(error, crate::ftps::config::FtpsInitError::InvalidConfig(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn protocol_client_ca_configuration_cannot_allow_cleartext() {
|
||||
let directory = tempfile::tempdir().expect("config directory");
|
||||
let ca_file = directory.path().join("clients.pem");
|
||||
std::fs::write(&ca_file, "test").expect("existing CA path");
|
||||
#[cfg(feature = "webdav")]
|
||||
{
|
||||
let config = crate::webdav::config::WebDavConfig {
|
||||
tls_enabled: false,
|
||||
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
|
||||
ca_file: Some(ca_file.to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(config.validate().await.is_err());
|
||||
}
|
||||
#[cfg(feature = "ftps")]
|
||||
{
|
||||
let config = crate::ftps::config::FtpsConfig {
|
||||
tls_enabled: false,
|
||||
cert_dir: Some(directory.path().to_string_lossy().into_owned()),
|
||||
ca_file: Some(ca_file.to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(config.validate().await.is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,12 +56,6 @@ impl FtpsConfig {
|
||||
));
|
||||
}
|
||||
|
||||
if self.ca_file.is_some() && (!self.tls_enabled || self.cert_dir.is_none()) {
|
||||
return Err(FtpsInitError::InvalidConfig(
|
||||
"A client CA requires TLS and a server certificate directory".to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(path) = &self.cert_dir
|
||||
&& !tokio::fs::try_exists(path).await.unwrap_or(false)
|
||||
{
|
||||
|
||||
@@ -110,9 +110,6 @@ where
|
||||
ftps_required = self.config.ftps_required,
|
||||
"FTPS server starting"
|
||||
);
|
||||
if self.config.tls_enabled && self.config.cert_dir.is_some() {
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
|
||||
}
|
||||
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
|
||||
|
||||
let storage_clone = self.storage.clone();
|
||||
@@ -121,7 +118,7 @@ where
|
||||
Arc::new(ConnectionUserDetailProvider),
|
||||
)
|
||||
.authenticator(Arc::new(ConnectionAuthenticator {
|
||||
require_data_tls: self.config.tls_enabled && (self.config.ftps_required || self.config.ca_file.is_some()),
|
||||
require_data_tls: self.config.tls_enabled && self.config.ftps_required,
|
||||
}));
|
||||
|
||||
// Configure passive ports for data connections
|
||||
@@ -193,12 +190,16 @@ where
|
||||
reload_shutdown_rx.clone(),
|
||||
);
|
||||
|
||||
let server_config = crate::common::tls::server_config(resolver, self.config.ca_file.as_deref())
|
||||
.map_err(|error| FtpsInitError::InvalidConfig(format!("Invalid client CA configuration: {error}")))?;
|
||||
// Build ServerConfig with SNI support
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let server_config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_cert_resolver(resolver);
|
||||
|
||||
server_builder = server_builder.ftps_manual::<std::path::PathBuf>(Arc::new(server_config));
|
||||
|
||||
if self.config.ftps_required || self.config.ca_file.is_some() {
|
||||
if self.config.ftps_required {
|
||||
debug!(
|
||||
event = EVENT_FTPS_TLS_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
@@ -208,7 +209,7 @@ where
|
||||
);
|
||||
server_builder = server_builder.ftps_required(FtpsRequired::All, FtpsRequired::All);
|
||||
}
|
||||
} else if self.config.ftps_required || self.config.ca_file.is_some() {
|
||||
} else if self.config.ftps_required {
|
||||
return Err(FtpsInitError::InvalidConfig(
|
||||
"FTPS required but certificate directory not provided".into(),
|
||||
));
|
||||
|
||||
@@ -43,70 +43,3 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
|
||||
|
||||
#[cfg(feature = "sftp")]
|
||||
pub use sftp::{SftpConfig, SftpInitError, SftpServer};
|
||||
|
||||
#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
|
||||
mod fips_tests {
|
||||
use crate::common::dummy_storage::DummyBackend;
|
||||
use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
|
||||
|
||||
#[test]
|
||||
fn protocols_reject_incompatible_host_provider() {
|
||||
const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
|
||||
if std::env::var_os(CHILD).is_some() {
|
||||
let mut provider = rustfs_tls_runtime::default_crypto_provider();
|
||||
provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
|
||||
provider.install_default().expect("install incompatible host provider");
|
||||
let cert_dir = tempfile::tempdir().expect("certificate directory");
|
||||
let cert_dir = cert_dir.path().to_string_lossy().into_owned();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
runtime.block_on(async {
|
||||
let ftps_config = FtpsConfig {
|
||||
cert_dir: Some(cert_dir.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
|
||||
.await
|
||||
.expect("FTPS config");
|
||||
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
|
||||
let error = ftps
|
||||
.start(receiver)
|
||||
.await
|
||||
.expect_err("FTPS must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
|
||||
let webdav_config = WebDavConfig {
|
||||
bind_addr: ([127, 0, 0, 1], 0).into(),
|
||||
cert_dir: Some(cert_dir),
|
||||
..Default::default()
|
||||
};
|
||||
let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
|
||||
.await
|
||||
.expect("WebDAV config");
|
||||
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
|
||||
let error = webdav
|
||||
.start(receiver)
|
||||
.await
|
||||
.expect_err("WebDAV must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
});
|
||||
return;
|
||||
}
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.arg("protocols_reject_incompatible_host_provider")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD, "1")
|
||||
.env("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||
.env("no_proxy", "localhost,127.0.0.1,::1")
|
||||
.output()
|
||||
.expect("start isolated protocol TLS test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,12 +65,6 @@ impl WebDavConfig {
|
||||
));
|
||||
}
|
||||
|
||||
if self.ca_file.is_some() && (!self.tls_enabled || self.cert_dir.is_none()) {
|
||||
return Err(WebDavInitError::InvalidConfig(
|
||||
"A client CA requires TLS and a server certificate directory".to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(path) = &self.cert_dir
|
||||
&& !tokio::fs::try_exists(path).await.unwrap_or(false)
|
||||
{
|
||||
|
||||
@@ -30,6 +30,7 @@ use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
|
||||
use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
|
||||
use rustfs_utils::MaskedAccessKey;
|
||||
use rustls::ServerConfig;
|
||||
use std::convert::Infallible;
|
||||
use std::io;
|
||||
use std::net::IpAddr;
|
||||
@@ -117,10 +118,6 @@ where
|
||||
"WebDAV server starting"
|
||||
);
|
||||
|
||||
if self.config.tls_enabled && self.config.cert_dir.is_some() {
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
|
||||
}
|
||||
|
||||
let listener = TcpListener::bind(self.config.bind_addr).await?;
|
||||
self.serve(listener, shutdown_rx).await
|
||||
}
|
||||
@@ -158,8 +155,9 @@ where
|
||||
reload_shutdown_rx.clone(),
|
||||
);
|
||||
|
||||
let server_config = crate::common::tls::server_config(resolver, self.config.ca_file.as_deref())
|
||||
.map_err(|error| WebDavInitError::Tls(format!("Invalid client CA configuration: {error}")))?;
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
|
||||
|
||||
Some(TlsAcceptor::from(Arc::new(server_config)))
|
||||
} else {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user