Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue 91f2d53500 fix(ecstore): order foreground writes during disk recovery 2026-11-02 06:19:44 +08:00
94 changed files with 12531 additions and 3398 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+10 -4
View File
@@ -417,6 +417,12 @@ threads-required = "num-test-threads"
filter = 'package(rustfs-heal) & (binary(mrf_partial_write_test) | (binary(mrf_pipeline_test) & test(/^journal_replay_survives_/)) | test(=admin_selector_survives_sigkill_with_original_scope_and_receipts))'
threads-required = "num-test-threads"
# Bound the whole recovery fixture even if a synchronous wait prevents its
# per-case async deadline from firing.
[[profile.ci.overrides]]
filter = 'package(rustfs-heal) & binary(heal_integration_test) & test(=serial_tests::test_admin_bucket_directory_recovery_from_task_entry)'
slow-timeout = { period = "60s", terminate-after = 5, grace-period = "10s" }
# Native scanner backlog fixtures include 24-set and pool-expansion cases.
# Reserve capacity across test processes while preserving each scenario's fixed
# budget, assertions, and internal concurrency.
@@ -827,7 +833,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
# ---------------------------------------------------------------------------
# e2e-distributed profile — 4-node 4-disk Actions suite
# ---------------------------------------------------------------------------
# Nightly and manual-dispatch lane owned by
# Storage-sensitive PR / nightly / dispatch lane owned by
# .github/workflows/e2e-distributed.yml.
# Each case starts four rustfs processes (and for site replication, two
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -970,10 +976,10 @@ threads-required = "num-test-threads"
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
test-group = 'e2e-vault'
# Keep the storage-heavy smoke cases isolated in the full suite too. They
# start multiple server fixtures and issue many sequential S3 operations.
# Keep storage-heavy cases isolated in the full suite too. The large pagination
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
threads-required = "num-test-threads"
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
-2
View File
@@ -32,8 +32,6 @@ updates:
- "overtrue"
- "majinghe"
ignore:
- dependency-name: "pulsar"
versions: [ "6.9.x" ]
- dependency-name: "object_store"
versions: [ "0.13.x" ]
- dependency-name: "libunftp"
+5 -2
View File
@@ -769,10 +769,12 @@
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
"roles": [
"pr",
"periodic",
"manual"
],
"events": [
"pull_request",
"workflow_dispatch",
"schedule"
],
@@ -847,11 +849,12 @@
"workflow": ".github/workflows/e2e-upgrade.yml",
"name": "Upgrade Compatibility",
"roles": [
"pr",
"periodic",
"manual",
"release"
"manual"
],
"events": [
"pull_request",
"push",
"schedule",
"workflow_dispatch"
+28 -2
View File
@@ -17,8 +17,8 @@
# Each selected test starts a real localhost cluster via
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
# and manual dispatches execute the same fail-closed suite.
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
# nightly runs, and manual dispatches all execute the same fail-closed suite.
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
#
# Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,6 +33,32 @@
name: e2e-distributed
on:
pull_request:
paths:
- "Cargo.lock"
- "Cargo.toml"
- ".config/nextest.toml"
- ".config/e2e-distributed-selection.txt"
- ".github/workflows/e2e-distributed.yml"
- "crates/audit/**"
- "crates/common/**"
- "crates/config/**"
- "crates/e2e_test/**"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/iam/**"
- "crates/lock/**"
- "crates/madmin/**"
- "crates/notify/**"
- "crates/replication/**"
- "crates/s3-client/**"
- "crates/s3-ops/**"
- "crates/s3-types/**"
- "crates/scanner/**"
- "crates/storage-api/**"
- "crates/utils/**"
- "rustfs/**"
workflow_dispatch:
inputs:
filter:
+18 -7
View File
@@ -12,19 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Single-node upgrade and rollback lane.
#
# The weekly Monday schedule, version tags, and workflow_dispatch run this
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
name: Upgrade Compatibility
on:
pull_request:
paths:
- ".github/workflows/e2e-upgrade.yml"
- "crates/e2e_test/src/common.rs"
- "crates/e2e_test/src/distributed/**"
- "crates/e2e_test/src/fake_s3_target/**"
- "crates/e2e_test/src/lib.rs"
- "crates/e2e_test/src/replication_extension_test.rs"
- "crates/e2e_test/src/rpc_fault_proxy.rs"
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
- "crates/ecstore/**"
- "crates/filemeta/**"
- "crates/heal/**"
- "crates/kms/**"
- "crates/storage-api/**"
- "rustfs/**"
- "Cargo.lock"
push:
tags:
- "[0-9]*.[0-9]*.[0-9]*"
schedule:
# 03:17 UTC every Monday.
- cron: "17 3 * * 1"
workflow_dispatch:
@@ -33,7 +44,7 @@ permissions:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
+4 -88
View File
@@ -346,12 +346,12 @@ jobs:
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
# engineers can download and install the nightly directly. The branch is
# a single-commit orphan rewritten on every build, which keeps the repo
# small while the latest files stay reachable at stable raw URLs.
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
# The dispatch below runs only after this upload and the immutable R2
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
# which still starts from this workflow's workflow_run completion.
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
- name: Publish packages to auto-testing release assets
env:
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,90 +360,6 @@ jobs:
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
run: bash scripts/release/publish_nightly_assets.sh
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
# or fine-grained Contents write (the permission GitHub checks for
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
# step, and therefore this job, so the scheduled failure alert fires.
# GitHub allows at most 10 top-level client_payload properties, so the
# schema 2 candidate object is nested under `candidate` and the release
# fields sit beside it.
- name: Dispatch nightly candidate to auto-testing
if: ${{ steps.publish.outputs.candidate_file != '' }}
env:
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
SOURCE_REPO: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
: "${DEB_DATE:?DEB_DATE is required}"
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
--arg event_type "nightly-candidate-ready" \
--arg source_repo "${SOURCE_REPO}" \
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
--arg channel "nightly" \
--arg deb_date "${DEB_DATE}" \
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
'{
event_type: $event_type,
source_repo: $source_repo,
latest_alias_url: $latest_alias_url,
auto_testing_release: $auto_testing_release,
channel: $channel,
deb_date: $deb_date,
nightly_workflow: $nightly_workflow,
candidate: $candidate[0]
}' > "${payload}"
if ! jq -e '
.event_type == "nightly-candidate-ready"
and .source_repo == "rustfs/rustfs"
and .channel == "nightly"
and .deb_date == env.DEB_DATE
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
and .candidate.schema == 2
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
' "${payload}" >/dev/null; then
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
exit 1
fi
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --slurpfile client_payload "${payload}" \
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
-o "${response}" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "Content-Type: application/json" \
--data-binary @"${body}" \
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
exit 1
}
if [[ "${status}" != "204" ]]; then
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
if [[ -s "${response}" ]]; then
cat "${response}" >&2
fi
exit 1
fi
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
+5 -41
View File
@@ -92,17 +92,6 @@ jobs:
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
run: |
set -euo pipefail
# Scheduled runs carry no inputs and the resolver only derives a
# build from a workflow_run payload (which a schedule cannot
# provide), so resolve the newest successful nightly build
# explicitly before invoking it.
if [ -z "${BUILD_RUN_ID:-}" ]; then
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
fi
python3 scripts/resolve_functional_candidate.py
python3 - <<'PY'
import json
@@ -110,15 +99,15 @@ jobs:
from pathlib import Path
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
root.mkdir(parents=True, exist_ok=True)
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = chain["candidate"]["manifest"]
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = candidate["manifest"]
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("testing_sha=" + chain["testing_sha"] + "\n")
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
output.write("package_url=" + manifest["package_url"] + "\n")
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
@@ -132,26 +121,6 @@ jobs:
path: auto-testing
persist-credentials: false
- name: Prepare validation Python runtime
# The pool / ilm / replication validation runners require a managed
# Python, exactly like the chain's validation job provides.
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install validation Python requirements
# Case scripts call python3 and import boto3 at startup. Install the
# functional chain's client into a per-run env so that interpreter
# sees it without changing the runner's shared Python.
run: |
set -euo pipefail
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
test ! -e "${validation_python}"
python3 -m venv "${validation_python}"
"${validation_python}/bin/python" -m pip install \
-r auto-testing/scripts/validation_ilm/requirements.txt
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
- name: Select sweep cases
id: select
env:
@@ -255,8 +224,7 @@ jobs:
# topology, this purge guarantees the next case starts clean even
# when the previous one died mid-run.
for node in ${RUSTFS_NODES}; do
# -n keeps ssh from consuming the case list this loop reads from stdin
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
@@ -275,7 +243,6 @@ jobs:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
run: |
set -euo pipefail
python3 - <<'PY'
@@ -384,10 +351,7 @@ jobs:
${{ runner.temp }}/validation-sweep/sweep-cases.txt
${{ runner.temp }}/validation-sweep/candidate.json
${{ runner.temp }}/validation-sweep/results/
${{ runner.temp }}/validation-sweep/run/
# warn (not error): an early infra failure legitimately leaves no
# per-case evidence, and the failing step is already red.
if-no-files-found: warn
if-no-files-found: error
retention-days: 30
- name: Cleanup environment (after)
Generated
+142 -341
View File
@@ -299,7 +299,7 @@ dependencies = [
"strum",
"thiserror 2.0.21",
"uuid",
"zstd 0.13.4",
"zstd 0.13.3",
]
[[package]]
@@ -370,17 +370,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df"
dependencies = [
"arrow-arith",
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-csv",
"arrow-data 59.3.2",
"arrow-ipc 59.3.2",
"arrow-data",
"arrow-ipc",
"arrow-json",
"arrow-ord",
"arrow-row",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-schema",
"arrow-select",
"arrow-string",
]
@@ -390,10 +390,10 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"num-traits",
]
@@ -405,9 +405,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43"
dependencies = [
"ahash",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"chrono-tz",
"half",
@@ -417,24 +417,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "arrow-array"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"
dependencies = [
"ahash",
"arrow-buffer 60.1.0",
"arrow-data 60.1.0",
"arrow-schema 60.1.0",
"chrono",
"half",
"hashbrown 0.17.1",
"num-complex",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "59.3.2"
@@ -447,30 +429,18 @@ dependencies = [
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"
dependencies = [
"bytes",
"half",
"num-bigint 0.5.1",
"num-traits",
]
[[package]]
name = "arrow-cast"
version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ord",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-schema",
"arrow-select",
"atoi 2.0.0",
"base64 0.23.1",
"chrono",
@@ -481,26 +451,15 @@ dependencies = [
"ryu",
]
[[package]]
name = "arrow-cmp"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"
dependencies = [
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-schema 60.1.0",
]
[[package]]
name = "arrow-csv"
version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698"
dependencies = [
"arrow-array 59.3.2",
"arrow-array",
"arrow-cast",
"arrow-schema 59.3.2",
"arrow-schema",
"chrono",
"csv",
"csv-core",
@@ -513,21 +472,8 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba"
dependencies = [
"arrow-buffer 59.3.2",
"arrow-schema 59.3.2",
"half",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-data"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"
dependencies = [
"arrow-buffer 60.1.0",
"arrow-schema 60.1.0",
"arrow-buffer",
"arrow-schema",
"half",
"num-integer",
"num-traits",
@@ -539,29 +485,14 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"flatbuffers",
"lz4_flex",
"zstd 0.13.4",
]
[[package]]
name = "arrow-ipc"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"
dependencies = [
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-data 60.1.0",
"arrow-schema 60.1.0",
"arrow-select 60.1.0",
"flatbuffers",
"zstd 0.14.3",
"zstd 0.13.3",
]
[[package]]
@@ -570,12 +501,12 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-ord",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-schema",
"arrow-select",
"chrono",
"half",
"indexmap 2.14.2",
@@ -595,11 +526,11 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
]
[[package]]
@@ -608,10 +539,10 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"half",
]
@@ -625,12 +556,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "arrow-schema"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"
[[package]]
name = "arrow-select"
version = "59.3.2"
@@ -638,25 +563,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4"
dependencies = [
"ahash",
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"num-traits",
]
[[package]]
name = "arrow-select"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"
dependencies = [
"ahash",
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-cmp",
"arrow-data 60.1.0",
"arrow-schema 60.1.0",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"num-traits",
]
@@ -666,11 +576,11 @@ version = "59.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470"
dependencies = [
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"memchr",
"num-traits",
"regex",
@@ -759,9 +669,9 @@ dependencies = [
[[package]]
name = "async-compression"
version = "0.4.52"
version = "0.4.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459"
checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970"
dependencies = [
"compression-codecs",
"compression-core",
@@ -998,28 +908,12 @@ dependencies = [
"zeroize",
]
[[package]]
name = "aws-lc-fips-sys"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b744eed18cd75d814effdf2aba2880f912abe0fe1c327b361341d31aba9a7005"
dependencies = [
"bindgen",
"cc",
"cmake",
"dunce",
"fs_extra",
"pkg-config",
"regex",
]
[[package]]
name = "aws-lc-rs"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53"
dependencies = [
"aws-lc-fips-sys",
"aws-lc-sys",
"untrusted 0.7.1",
"zeroize",
@@ -1066,9 +960,9 @@ dependencies = [
[[package]]
name = "aws-sdk-kms"
version = "1.129.0"
version = "1.128.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5cdb29ad25d71c9ee54c408a672c98d90a24a0ea6eb063025483e0e894b411"
checksum = "ef519fda62fd0fdc066948f5ef33ecb8ae1bf0187bf1a12e8a775c93c4c47e7e"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1091,9 +985,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.157.0"
version = "1.156.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8"
checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1127,9 +1021,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
version = "1.119.0"
version = "1.118.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c414f25ec4aa9dc18f5cc8f55e18848bb509fcda4ce7c77bee6072e95806bc9"
checksum = "b4a89786db8ee6dff2c9302a354ef081354fcb0f527e16ee3cb249a6a5cb6df7"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1152,9 +1046,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
version = "1.121.0"
version = "1.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc638bb4355e6514334dcc3228cd4622f5ee54d704bd6a9010a5a91d7856d9d9"
checksum = "521c6a11a25f1244efddf947523f277a51a63f0a7ec228f0f62d826e88106366"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1177,9 +1071,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
version = "1.124.0"
version = "1.123.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f8a0dd8fbde04ba58ad722113c7addb11bf94cf549b1ea0eb9ea427fb87198a"
checksum = "8754d38d29448bd1d4aa4a6312ed010dfa296b3436eb2c551cc6e42fc6a66a38"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1630,8 +1524,6 @@ dependencies = [
"bitflags 2.13.2",
"cexpr",
"clang-sys",
"log",
"prettyplease 0.3.0",
"proc-macro2",
"quote",
"regex",
@@ -2083,9 +1975,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.11.0"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b"
checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2341,9 +2233,9 @@ dependencies = [
[[package]]
name = "compression-codecs"
version = "0.4.47"
version = "0.4.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43"
checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
dependencies = [
"brotli 9.1.1",
"bzip2",
@@ -2589,9 +2481,9 @@ dependencies = [
[[package]]
name = "crc32fast"
version = "1.5.4"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04"
checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
dependencies = [
"cfg-if",
]
@@ -2981,7 +2873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"chrono",
"datafusion-catalog",
@@ -3015,7 +2907,7 @@ dependencies = [
"log",
"object_store",
"parking_lot 0.12.5",
"parquet 59.3.2",
"parquet",
"sqlparser",
"tempfile",
"tokio",
@@ -3079,8 +2971,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687"
dependencies = [
"arrow",
"arrow-ipc 59.3.2",
"arrow-schema 59.3.2",
"arrow-ipc",
"arrow-schema",
"chrono",
"foldhash 0.2.0",
"half",
@@ -3091,7 +2983,7 @@ dependencies = [
"log",
"num-traits",
"object_store",
"parquet 59.3.2",
"parquet",
"recursive",
"sqlparser",
"tokio",
@@ -3147,7 +3039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb"
dependencies = [
"arrow",
"arrow-ipc 59.3.2",
"arrow-ipc",
"async-trait",
"bytes",
"datafusion-common",
@@ -3217,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"bytes",
"datafusion-common",
@@ -3238,7 +3130,7 @@ dependencies = [
"log",
"object_store",
"parking_lot 0.12.5",
"parquet 59.3.2",
"parquet",
"tokio",
]
@@ -3255,7 +3147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837"
dependencies = [
"arrow",
"arrow-buffer 59.3.2",
"arrow-buffer",
"async-trait",
"bytes",
"dashmap",
@@ -3281,7 +3173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"chrono",
"datafusion-common",
@@ -3317,7 +3209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771"
dependencies = [
"arrow",
"arrow-buffer 59.3.2",
"arrow-buffer",
"base64 0.23.1",
"chrono",
"chrono-tz",
@@ -3477,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406"
dependencies = [
"arrow",
"arrow-schema 59.3.2",
"arrow-schema",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
@@ -3552,10 +3444,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97"
dependencies = [
"arrow",
"arrow-data 59.3.2",
"arrow-ipc 59.3.2",
"arrow-data",
"arrow-ipc",
"arrow-ord",
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"bytes",
"datafusion-common",
@@ -3603,7 +3495,7 @@ version = "55.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a"
dependencies = [
"arrow-schema 59.3.2",
"arrow-schema",
"async-trait",
"datafusion-common",
"datafusion-execution",
@@ -4050,7 +3942,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid 0.9.6",
"crypto-common 0.1.7",
"subtle",
]
@@ -5386,9 +5277,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.30.3"
version = "0.30.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935"
checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e"
dependencies = [
"arc-swap",
"async-channel",
@@ -5421,15 +5312,15 @@ dependencies = [
[[package]]
name = "hotpath-drain"
version = "0.30.3"
version = "0.30.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "738324f75aec847dea984f6806402f4f1bdfab437b4700fc8c3ebcad2dff6424"
checksum = "d0faae1a9e1fd8ce5debaaaf452dcbe833fc8a8a4f3779d2ec8828b116c3c16a"
[[package]]
name = "hotpath-macros"
version = "0.30.3"
version = "0.30.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a"
checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58"
dependencies = [
"proc-macro2",
"quote",
@@ -6050,7 +5941,7 @@ dependencies = [
"lz4",
"snap",
"uuid",
"zstd 0.13.4",
"zstd 0.13.3",
]
[[package]]
@@ -6143,9 +6034,6 @@ name = "lazy_static"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
dependencies = [
"spin 0.9.9",
]
[[package]]
name = "lexical-core"
@@ -6355,9 +6243,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.35"
version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
[[package]]
name = "lru"
@@ -6721,9 +6609,9 @@ dependencies = [
[[package]]
name = "ml-kem"
version = "0.3.4"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84ca78b66f46050d49fa21693ae8adab2081d8101b65d8067316b6b70b64a0d1"
checksum = "a9ba01ebcf90f6aff8e9b9f55bb69b19e6250e4ade5787d0040eaa27dd226238"
dependencies = [
"hybrid-array",
"kem",
@@ -6736,9 +6624,9 @@ dependencies = [
[[package]]
name = "module-lattice"
version = "0.2.4"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e412c8ee83a1c678f427806219c49c96b2511b9f07497514f0e43fb26155"
checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe"
dependencies = [
"ctutils",
"hybrid-array",
@@ -7034,22 +6922,6 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "num-bigint-dig"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7"
dependencies = [
"lazy_static",
"libm",
"num-integer",
"num-iter",
"num-traits",
"rand 0.8.8",
"smallvec",
"zeroize",
]
[[package]]
name = "num-complex"
version = "0.4.6"
@@ -7074,16 +6946,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
@@ -7155,19 +7017,18 @@ dependencies = [
[[package]]
name = "oauth2"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
version = "5.1.0"
source = "git+https://github.com/ramosbugs/oauth2-rs?rev=89585f4a280afd7b285eaf1c9be0068874e03d22#89585f4a280afd7b285eaf1c9be0068874e03d22"
dependencies = [
"base64 0.22.1",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
"rand 0.8.8",
"rand 0.10.3",
"serde",
"serde_json",
"serde_path_to_error",
"sha2 0.10.9",
"sha2 0.11.1",
"thiserror 1.0.69",
"url",
]
@@ -7329,31 +7190,29 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "openidconnect"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0d098a59353679624d406825940139e2b2d554a52b64c9fb1b1bc42a11fe5c"
version = "4.0.1"
source = "git+https://github.com/ramosbugs/openidconnect-rs?rev=2f77ee73bc4554e1cb6279adb386005429fb4e41#2f77ee73bc4554e1cb6279adb386005429fb4e41"
dependencies = [
"base64 0.22.1",
"chrono",
"dyn-clone",
"ed25519-dalek 3.0.0",
"getrandom 0.4.3",
"hmac 0.12.1",
"hmac 0.13.0",
"http 1.5.0",
"itertools 0.14.0",
"log",
"oauth2",
"p256",
"p384",
"rand 0.8.8",
"rsa 0.9.10",
"rand 0.10.3",
"rsa",
"serde",
"serde-value",
"serde_json",
"serde_path_to_error",
"serde_plain",
"serde_with",
"sha2 0.10.9",
"sha2 0.11.1",
"subtle",
"thiserror 1.0.69",
"url",
@@ -7681,12 +7540,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680"
dependencies = [
"ahash",
"arrow-array 59.3.2",
"arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-ipc 59.3.2",
"arrow-schema 59.3.2",
"arrow-select 59.3.2",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ipc",
"arrow-schema",
"arrow-select",
"base64 0.23.1",
"brotli 8.0.4",
"bytes",
@@ -7705,33 +7564,7 @@ dependencies = [
"snap",
"tokio",
"twox-hash",
"zstd 0.13.4",
]
[[package]]
name = "parquet"
version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"
dependencies = [
"ahash",
"arrow-array 60.1.0",
"arrow-buffer 60.1.0",
"arrow-data 60.1.0",
"arrow-ipc 60.1.0",
"arrow-schema 60.1.0",
"arrow-select 60.1.0",
"base64 0.23.1",
"bytes",
"chrono",
"half",
"hashbrown 0.17.1",
"num-bigint 0.5.1",
"num-integer",
"num-traits",
"seq-macro",
"twox-hash",
"zstd 0.14.3",
"zstd 0.13.3",
]
[[package]]
@@ -7983,17 +7816,6 @@ dependencies = [
"futures-io",
]
[[package]]
name = "pkcs1"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
dependencies = [
"der 0.7.10",
"pkcs8 0.10.2",
"spki 0.7.3",
]
[[package]]
name = "pkcs1"
version = "0.8.0-rc.4"
@@ -9092,9 +8914,9 @@ dependencies = [
[[package]]
name = "reflink-copy"
version = "0.1.32"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad64f61357bcf16105a0f353f9783733409c8bd889bc326ad857959605ea927"
checksum = "d9dd7ab4af0363d5ccfd2838d782a28196cf32a5cc2e4fe3c5dc83f2be588b8b"
dependencies = [
"cfg-if",
"libc",
@@ -9256,26 +9078,6 @@ dependencies = [
"xmlparser",
]
[[package]]
name = "rsa"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d"
dependencies = [
"const-oid 0.9.6",
"digest 0.10.7",
"num-bigint-dig",
"num-integer",
"num-traits",
"pkcs1 0.7.5",
"pkcs8 0.10.2",
"rand_core 0.6.4",
"signature 2.2.0",
"spki 0.7.3",
"subtle",
"zeroize",
]
[[package]]
name = "rsa"
version = "0.10.0-rc.18"
@@ -9286,7 +9088,7 @@ dependencies = [
"crypto-bigint",
"crypto-primes",
"digest 0.11.3",
"pkcs1 0.8.0-rc.4",
"pkcs1",
"pkcs8 0.11.0",
"rand_core 0.10.1",
"sha2 0.11.1",
@@ -9391,13 +9193,13 @@ dependencies = [
"p521",
"pageant",
"pbkdf2 0.13.0",
"pkcs1 0.8.0-rc.4",
"pkcs1",
"pkcs5 0.8.1",
"pkcs8 0.11.0",
"polyval",
"rand 0.10.3",
"rand_core 0.10.1",
"rsa 0.10.0-rc.18",
"rsa",
"russh-cryptovec",
"russh-util",
"salsa20 0.11.0",
@@ -9589,7 +9391,7 @@ dependencies = [
"regex",
"reqwest",
"rmp-serde",
"rsa 0.10.0-rc.18",
"rsa",
"rust-embed",
"rustfs-audit",
"rustfs-common",
@@ -9792,7 +9594,7 @@ dependencies = [
"jsonwebtoken",
"pbkdf2 0.13.0",
"rand 0.10.3",
"rsa 0.10.0-rc.18",
"rsa",
"serde",
"serde_json",
"sha2 0.11.1",
@@ -10330,18 +10132,18 @@ dependencies = [
[[package]]
name = "rustfs-mimalloc"
version = "0.7.3"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216ecfc9341b5d6e2506d47ce57c9232ae1b8020bb10264ea05286fb234286e7"
checksum = "8301a578772fc9823d68e083e42c3b414613d588a0eac594c1ca6bc0bbe0653f"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.7.3"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb16ad68f96883407a75cdb88252b43dc2ea592428f0fc54ff7885756de4b814"
checksum = "6eb4082853043cb29556b4b776b05f28c3146706f43c53869dbf1ef262e00e91"
dependencies = [
"cc",
]
@@ -12186,7 +11988,7 @@ dependencies = [
"p384",
"p521",
"rand_core 0.10.1",
"rsa 0.10.0-rc.18",
"rsa",
"sec1",
"serde",
"sha1 0.11.0",
@@ -12536,18 +12338,18 @@ dependencies = [
[[package]]
name = "test-case"
version = "3.4.1"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae6e80608d2037173d7d59a09089d249e43352c182f6dde6b00d138bcb7cfdbe"
checksum = "124953e7f67cb0b2fcfb87e899e4ae5a64fb68e2e160767933cc67b646fb0042"
dependencies = [
"test-case-macros",
]
[[package]]
name = "test-case-core"
version = "3.4.1"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29b8e1f65819001927b8cdcbb5560f2428212f2e5ab5684e3c6815a4d80e72db"
checksum = "bd097615b407247e102046ba34426bb594923ab3554cf0f3cdfed050f1a5a3e8"
dependencies = [
"cfg-if",
"proc-macro2",
@@ -12557,9 +12359,9 @@ dependencies = [
[[package]]
name = "test-case-macros"
version = "3.4.1"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "802d6165a14df1598b7290a8c546df402940d3cfbe82d15c0fd35de176771693"
checksum = "f4ce35ece947bccac166e1ded639133827a953f69a48ba134d17f91108333082"
dependencies = [
"proc-macro2",
"quote",
@@ -13294,9 +13096,9 @@ dependencies = [
[[package]]
name = "unicode-properties"
version = "0.1.5"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833"
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
[[package]]
name = "unicode-segmentation"
@@ -14198,9 +14000,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.2.2"
version = "9.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e408c6cc4699dd017d53dc080cd51c2cea2b5354af57a1479bd265ed42911487"
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
dependencies = [
"aes 0.9.3",
"bzip2",
@@ -14211,7 +14013,6 @@ dependencies = [
"getrandom 0.4.3",
"hmac 0.13.0",
"indexmap 2.14.2",
"libc",
"lzma-rust2",
"memchr",
"nix 0.31.3",
@@ -14252,15 +14053,15 @@ dependencies = [
[[package]]
name = "zstd"
version = "0.13.4"
version = "0.13.3"
dependencies = [
"apache-avro",
"arrow-array 60.1.0",
"arrow-ipc 60.1.0",
"arrow-schema 60.1.0",
"arrow-array",
"arrow-ipc",
"arrow-schema",
"bytes",
"kafka-protocol",
"parquet 60.1.0",
"parquet",
"semver",
"toml",
"zstd 0.14.3",
+13 -9
View File
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-compression = { version = "0.4.51" }
async-trait = "0.1.92"
async-nats = { version = "0.51.0", default-features = false }
axum = "0.8.9"
@@ -217,7 +217,7 @@ chacha20poly1305 = { version = "0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.1.0" }
openidconnect = { default-features = false, version = "4.1.0" }
openidconnect = { default-features = false, version = "4.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
@@ -253,9 +253,9 @@ atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.0" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.129.0" }
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
aws-sdk-sts = { default-features = false, version = "1.124.0" }
aws-sdk-kms = { default-features = false, version = "1.128.0" }
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
aws-sdk-sts = { default-features = false, version = "1.123.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.19.2" }
@@ -337,7 +337,7 @@ strum = { version = "0.28.0" }
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
test-case = "3.4.0"
thiserror = "2.0.21"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
@@ -356,7 +356,7 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.2.2"
zip = "9.1.3"
zstd = "0.14.3"
# Observability and Metrics
@@ -384,9 +384,9 @@ russh-sftp = "3.0.1"
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.3" }
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.3", default-features = false }
hotpath = { version = "0.30.2", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
@@ -427,3 +427,7 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
zstd = { path = "crates/zstd-compat" }
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
+3 -3
View File
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -12,249 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
use crate::common::init_logging;
use aws_sdk_s3::{
Client,
error::{ProvideErrorMetadata, SdkError},
primitives::ByteStream,
};
use std::any::TypeId;
use std::collections::BTreeSet;
use std::fmt::Debug;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Barrier;
use tokio::time::Instant;
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
where
E: Debug + ProvideErrorMetadata + 'static,
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, SdkError<E>>>,
{
let started = Instant::now();
let mut attempts = 0;
// The workload uses unique unversioned keys. Repeat only explicitly
// advertised lock backpressure; every step still needs its real result.
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
loop {
attempts += 1;
let result = call().await;
let retry = result.as_ref().err().is_some_and(|error| {
let Some(response) = error.raw_response() else {
return false;
};
let Some(service_error) = error.as_service_error() else {
return false;
};
// HEAD has no XML error body, so its SDK service error can
// lack a code even when the server supplies retry headers.
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|| (service_error.code().is_none()
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
response.status().as_u16() == 503
&& namespace_code
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
&& response.headers().get("retry-after") == Some("1")
});
if !retry {
return result;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
})
.await;
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
eprintln!(
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
started.elapsed()
);
}
outcome.map_err(|_| {
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
})
}
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
let actual = output.body.collect().await?.into_bytes();
if actual.as_ref() != expected {
return Err(format!(
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
expected.len(),
sha256_hex(expected),
actual.len(),
sha256_hex(&actual)
)
.into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_http_client::test_util::infallible_client_fn;
use aws_smithy_types::body::SdkBody;
use std::sync::atomic::{AtomicUsize, Ordering};
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
let requests = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&requests);
let transport = infallible_client_fn(move |_| {
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
let body = if code.is_empty() {
body.to_string()
} else {
format!("<Error><Code>{code}</Code></Error>")
};
let mut response = http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len());
if let Some(cause) = cause {
response = response.header("x-rustfs-error-cause", cause);
}
if let Some(retry_after) = retry_after {
response = response.header("retry-after", retry_after);
}
response.body(SdkBody::from(body)).expect("scripted S3 response")
});
let config = crate::common::build_test_s3_config(
"http://contention.example.invalid",
"synthetic-access",
"synthetic-secret",
None,
"contention-test",
);
let client = Client::from_conf(config.to_builder().http_client(transport).build());
(s3_probe_client(&client), requests)
}
fn contention() -> Reply {
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
}
#[tokio::test(start_paused = true)]
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt < 3 {
contention()
} else {
(200, "", None, None, "")
}
});
let standard = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
.build(),
);
standard
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
.await
.expect_err("three advertised lock responses exhaust the standard SDK policy");
assert_eq!(requests.load(Ordering::SeqCst), 3);
requests.store(0, Ordering::SeqCst);
let started = Instant::now();
after_lock_contention("PUT recovery", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await??;
assert_eq!(requests.load(Ordering::SeqCst), 4);
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
Ok(())
}
#[tokio::test(start_paused = true)]
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
let (client, requests) = client(|attempt| {
if attempt == 0 {
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
} else {
(200, "", None, None, "")
}
});
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
assert_eq!(requests.load(Ordering::SeqCst), 2);
Ok(())
}
#[tokio::test(start_paused = true)]
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
for reply in [
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(403, "AccessDenied", None, None, ""),
(404, "NoSuchKey", None, None, ""),
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
(503, "ServiceUnavailable", None, Some("1"), ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
] {
let (client, requests) = client(move |_| reply);
let result = after_lock_contention("PUT fatal response", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await?;
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn persistent_lock_contention_exhausts_the_request_budget() {
let (client, requests) = client(|_| contention());
let started = Instant::now();
let result = after_lock_contention("PUT unavailable", || {
client
.put_object()
.bucket("bucket")
.key("key")
.body(ByteStream::from_static(b"body"))
.send()
})
.await;
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
assert!(requests.load(Ordering::SeqCst) > 1);
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
}
#[tokio::test(start_paused = true)]
async fn acknowledged_wrong_bytes_fail_without_retry() {
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
.await
.expect_err("a successful but incorrect read is a consistency failure");
assert!(error.to_string().contains("bytes mismatch"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let dist = DistCluster::start(DistLayout::FourByFour).await?;
let bucket = unique_bucket("conc");
dist.create_bucket(&bucket).await?;
// This consistency probe owns its admission policy; unrelated responses
// must not be hidden by the SDK's general-purpose retries.
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
let clients = Arc::new(dist.clients()?);
let worker_count = 24;
let rounds = 4;
let barrier = Arc::new(Barrier::new(worker_count));
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
let key = format!("source/worker-{idx:02}-round-{round}.bin");
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
let body = payload_for(&key, 64 * 1024);
let context = format!("worker={idx} round={round}");
after_lock_contention(&format!("{context} PUT source"), || {
writer
.put_object()
.bucket(&bucket)
.key(&key)
.body(ByteStream::from(body.clone()))
.send()
})
.await??;
put_object(writer, &bucket, &key, body.clone()).await?;
let head = after_lock_contention(&format!("{context} HEAD source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await??;
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
if head.content_length() != Some(body.len() as i64) {
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
}
assert_workload_bytes(reader, &bucket, &key, &body).await?;
assert_object_bytes(reader, &bucket, &key, &body).await?;
after_lock_contention(&format!("{context} COPY retained"), || {
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
})
.await??;
assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
copier
.copy_object()
.bucket(&bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/{key}"))
.send()
.await?;
assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
after_lock_contention(&format!("{context} DELETE source"), || {
writer.delete_object().bucket(&bucket).key(&key).send()
})
.await??;
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
reader.head_object().bucket(&bucket).key(&key).send()
})
.await?
.expect_err("deleted source key must not remain visible");
writer.delete_object().bucket(&bucket).key(&key).send().await?;
let missing = reader
.head_object()
.bucket(&bucket)
.key(&key)
.send()
.await
.expect_err("deleted source key must not remain visible");
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
}
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
for handle in handles {
inventory.extend(handle.await??);
}
assert_eq!(
inventory.len(),
worker_count * rounds,
"every worker round must retain its acknowledged copy"
);
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
for (node_idx, client) in clients.iter().enumerate() {
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
})
.await??;
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
for (key, body) in &inventory {
assert_workload_bytes(client, &bucket, key, body)
assert_object_bytes(client, &bucket, key, body)
.await
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
}
@@ -576,6 +576,41 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
async fn reserve_object_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn reserve_object_metadata_short(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn activate_object_metadata_reservation(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn cancel_object_metadata_reservation(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_reserved_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_data_with_pool_meta_cas(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
+319 -7
View File
@@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
MetadataReservationCancelRequest, MetadataReservationRequest,
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -2452,6 +2453,239 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
}
impl RemoteDisk {
/// Prepare a remote token without retrying the RPC. It owns the bucket
/// publication fence but does not enter the object metadata queue yet.
pub async fn reserve_object_metadata(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
) -> Result<Uuid> {
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, false)
.await
}
/// Use the distinct short-lease RPC; an old peer rejects it before allocating a token.
pub(crate) async fn reserve_object_metadata_short(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
) -> Result<Uuid> {
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, true)
.await
}
async fn reserve_object_metadata_with_protocol(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
short: bool,
) -> Result<Uuid> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
self.execute_with_timeout_for_op_and_health_action(
"reserve_object_metadata",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationRequest {
disk: self.endpoint.to_string(),
bucket: bucket.to_owned(),
object: object.to_owned(),
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
.map_err(|_| Error::other("reservation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let result = if short {
client.reserve_object_metadata_short(request).await
} else {
client.reserve_object_metadata(request).await
};
let response = result
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
if response.protocol_version != if short { 3 } else { 2 } {
return Err(DiskError::UnsupportedDisk);
}
Uuid::from_slice(&response.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or(DiskError::Unexpected)
},
timeout.min(Duration::from_secs(5)),
FailureHealthAction::IgnoreFailure,
)
.await
}
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
/// result before allowing any subsequent unprotected mutation.
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
self.update_object_metadata_reservation(bucket, object, token, timeout, false)
.await
}
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
self.update_object_metadata_reservation(bucket, object, token, timeout, true)
.await
}
async fn update_object_metadata_reservation(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: Duration,
renew_only: bool,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
self.execute_with_timeout_for_op_and_health_action(
"activate_object_metadata_reservation",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationActivateRequest {
disk: self.endpoint.to_string(),
bucket: bucket.to_owned(),
object: object.to_owned(),
token: token.as_bytes().to_vec().into(),
renew_only,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
.map_err(|_| Error::other("reservation activation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.activate_object_metadata_reservation(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
Ok(())
},
timeout.min(Duration::from_secs(5)),
FailureHealthAction::IgnoreFailure,
)
.await
}
/// Cancel an unclaimed remote reservation. False cannot prove that an
/// earlier claim did not start a physical mutation.
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
self.execute_with_timeout_for_op(
"cancel_object_metadata_reservation",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationCancelRequest {
disk: self.endpoint.to_string(),
token: token.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.cancel_object_metadata_reservation(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if let Some(error) = response.error {
return Err(error.into());
}
Ok(response.success)
},
get_max_timeout_duration(),
)
.await
}
/// Publish a staged user-object write using a remote one-shot reservation.
/// The caller must retain the bucket lifecycle fence until the tail drains.
pub async fn rename_data_with_metadata_reservation(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
token: Uuid,
) -> Result<RenameDataResp> {
self.execute_with_timeout_for_op(
"rename_reserved_data",
|| async {
let mut client = self.get_client().await?;
let rename = RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_owned(),
src_path: src_path.to_owned(),
file_info: compat_json(fi)?,
dst_volume: dst_volume.to_owned(),
dst_path: dst_path.to_owned(),
file_info_bin: encode_file_info_msgpack(fi)?.into(),
scanner_publication_lease_token: Bytes::new(),
bucket_incarnation_id: Bytes::new(),
};
let mut request = Request::new(ReservedRenameDataRequest {
rename: Some(rename),
token: token.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
.map_err(|_| Error::other("reserved rename request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.rename_reserved_data(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
decode_msgpack_or_json::<RenameDataResp>(
&response.rename_data_resp_bin,
&response.rename_data_resp,
"RenameDataResp",
)
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed(
&self,
@@ -5109,6 +5343,84 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
async fn short_metadata_reservation_rejects_old_peer_without_legacy_rpc() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
assert!(matches!(
remote
.reserve_object_metadata_short("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
vec!["/node_service.NodeService/ReserveObjectMetadataShort"],
"an old peer must never receive the longer-lived v2 request"
);
peer.stop().await;
}
#[tokio::test]
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
let token = Uuid::new_v4();
assert!(matches!(
remote
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.renew_object_metadata("bucket", "object", token, Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote.cancel_object_metadata_reservation(token).await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.rename_data_with_metadata_reservation(
crate::disk::RUSTFS_META_TMP_BUCKET,
"staged",
&sample_rename_data_file_info(),
"bucket",
"object",
token,
)
.await,
Err(DiskError::UnsupportedDisk)
));
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
vec![
"/node_service.NodeService/ReserveObjectMetadata",
"/node_service.NodeService/ActivateObjectMetadataReservation",
"/node_service.NodeService/ActivateObjectMetadataReservation",
"/node_service.NodeService/CancelObjectMetadataReservation",
"/node_service.NodeService/RenameReservedData",
],
"reservation operations must never downgrade to legacy methods",
);
peer.stop().await;
}
#[tokio::test]
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -1315,6 +1315,122 @@ impl LockClient for RemoteClient {
Ok(resp.success)
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
// Older nodes refresh normally but return no conversion marker.
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
if !response.success {
return Ok(None);
}
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
Some(":0") => Some(false),
Some(":1") => Some(true),
_ => None,
})
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
File diff suppressed because it is too large Load Diff
+13
View File
@@ -655,6 +655,19 @@ impl Sets {
.await
}
pub(crate) async fn put_object_with_prepared_reservations(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
self.get_disks_by_key(object)
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
.await
}
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
&self,
bucket: &str,
+295 -124
View File
@@ -604,8 +604,69 @@ impl LocalDiskWrapper {
let super::RenameDataGuards {
external_guard,
namespace_owner,
metadata_reservation,
metadata_reservation_token,
..
} = guards;
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
(Some(reservation), None) => Some(reservation),
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
Ok(reservation) => Some(reservation),
// A missing token may already have been claimed by an earlier
// request that is still publishing; do not claim it was safe.
Err(DiskError::DiskOngoingReq) => {
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
}
Err(error) => return super::RenameDataObservation::not_dispatched(error),
},
(None, None) => None,
};
// A queued reservation must acquire its metadata lease before disk
// admission. Otherwise a waiting background mutation could hold the
// disk permit needed by the earlier lease owner.
let metadata_reservation = match metadata_reservation {
Some(reservation) => Some(reservation.wait().await),
None => None,
};
// The claimed reservation stays in this cancellable future while disk
// admission can wait. Only physical work moves into an owned task.
let prechecked_permit = if metadata_reservation.is_some() {
// Background admission is exclusive. Waiting for it while holding
// this object's lease can deadlock an admitted mutation waiting for
// the same lease. An already owned permit does not need to wait.
if current_disk_io_context().is_some_and(|context| {
context.class == DiskIoClass::Background
&& !context
.permit
.as_ref()
.is_some_and(|permit| permit.is_for(&self.io_admission))
}) {
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
}
let preflight = self
.with_native_resources(async {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.acquire_disk_io_permit().await
})
.await;
match preflight {
Ok(Ok(Some(permit))) => Some(permit),
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
}
} else {
None
};
// The reserved metadata lease must outlive a cancelled RPC waiter.
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
Some(Arc::new(external_guard))
} else {
external_guard
};
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
@@ -624,27 +685,54 @@ impl LocalDiskWrapper {
};
let observed = run_owned_mutation(external_guard, move || async move {
let mut preflight_rejection = None;
let result = operation
.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed = Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
let rename = || async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed = if let Some(reservation) = metadata_reservation {
Box::pin(operation.disk.rename_data_observed_with_reservation(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
(namespace_owner, Some(reservation)),
))
.await
} else {
Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
))
.await
};
preflight_rejection = observed.preflight_rejection;
observed.result
};
let result = if let Some(permit) = prechecked_permit {
with_disk_io_permit(Some(permit), async {
operation
.with_native_resources(operation.track_admitted_disk_operation(
"rename_data",
move |io_permit| async move {
let _io_permit = io_permit;
rename().await
},
Duration::ZERO,
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::Write,
true,
))
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
},
timeout_duration,
)
.await;
.await?
})
.await
} else {
operation
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
.await
};
// Health tracking must observe the real disk error, not an Ok tuple.
Ok(super::RenameDataObservation {
result,
@@ -853,8 +941,6 @@ pub(crate) struct DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
#[cfg(test)]
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
/// Caller API operations currently executing through the disk health wrapper.
api_waiting: AtomicU32,
/// Operations rejected because the disk was unavailable for the caller.
@@ -882,8 +968,6 @@ impl Default for DiskHealthMetricEpoch {
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
#[cfg(test)]
multipart_cleanup_observer: std::sync::Mutex::new(None),
#[cfg(test)]
operation_timeout_override: std::sync::Mutex::new(None),
api_waiting: AtomicU32::new(0),
total_errors_availability: AtomicU64::new(0),
total_errors_timeout: AtomicU64::new(0),
@@ -1577,20 +1661,6 @@ impl Default for DiskHealthTracker {
}
}
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct ScopedDiskOperationTimeout {
metrics: Arc<DiskHealthMetricEpoch>,
previous: Option<Duration>,
}
#[cfg(test)]
impl Drop for ScopedDiskOperationTimeout {
fn drop(&mut self) {
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
}
}
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
/// This is similar to Go's xlStorageDiskIDCheck.
#[derive(Debug, Clone)]
@@ -1616,6 +1686,81 @@ pub struct LocalDiskWrapper {
}
impl LocalDiskWrapper {
/// Reserve the metadata mutation lock for one object on this disk instance.
/// Unclaimed reservations expire after 60 seconds.
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.reserve_object_metadata(bucket, object, timeout).await
}
/// Register an object metadata reservation in FIFO order on this disk.
/// The returned token may be claimed before its physical lease is available.
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.enqueue_object_metadata(bucket, object).await
}
/// Prepare an inert reservation token bound to this disk and object.
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
self.prepare_object_metadata_with_owner(bucket, object, None).await
}
pub(crate) async fn prepare_object_metadata_with_owner(
&self,
bucket: &str,
object: &str,
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
}
pub(crate) async fn prepare_object_metadata_with_owner_ttl(
&self,
bucket: &str,
object: &str,
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
ttl: Duration,
) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
}
/// Refresh an inert token while the caller is still staging the write.
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.renew_object_metadata(bucket, object, token)
}
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.activate_object_metadata(bucket, object, token).await
}
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
self.disk.cancel_object_metadata_reservation(token)
}
/// Create a new LocalDiskWrapper
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
@@ -1932,23 +2077,6 @@ impl LocalDiskWrapper {
self.health_check
}
#[cfg(test)]
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
// The metrics epoch follows this drive across clones and reconnects.
// Its scoped override cannot shorten unrelated healthy drives' I/O.
let previous = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.replace(timeout);
ScopedDiskOperationTimeout {
metrics: Arc::clone(&self.metrics),
previous,
}
}
/// Enable health monitoring after disk creation.
/// Used to defer health checks until after startup format loading completes.
pub fn enable_health_check(&self) {
@@ -2616,6 +2744,7 @@ impl LocalDiskWrapper {
timeout_duration,
timeout_health_action,
mutation,
false,
))
.await?
}
@@ -2627,6 +2756,7 @@ impl LocalDiskWrapper {
timeout_duration: Duration,
timeout_health_action: TimeoutHealthAction,
mutation: DiskMetricMutation,
identity_prechecked: bool,
) -> Result<T>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
@@ -2648,7 +2778,7 @@ impl LocalDiskWrapper {
}
// Check if disk is stale
if let Err(err) = self.check_disk_stale().await {
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
self.metrics.record_availability_error();
return Err(err);
}
@@ -2673,13 +2803,6 @@ impl LocalDiskWrapper {
}
return result;
}
#[cfg(test)]
let timeout_duration = self
.metrics
.operation_timeout_override
.lock()
.expect("test timeout lock")
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
// Execute the operation with timeout
let result = tokio::time::timeout(timeout_duration, operation).await;
@@ -5623,6 +5746,115 @@ mod tests {
drop(metadata_lease);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
let dir = tempfile::tempdir().expect("disk directory");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let meta = dir.path().join(RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
.await
.expect("write format");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
disk.make_volume("bucket").await.expect("bucket volume");
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
let token = wrapper
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
.await
.expect("reserve key");
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
disk.format_info.write().await.last_check = None;
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
disk.install_format_probe_hook(false, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv_timeout(Duration::from_secs(30));
});
let publisher = Arc::clone(&wrapper);
let rename = tokio::spawn(async move {
publisher
.rename_data_observed_with_guards(
RUSTFS_META_TMP_BUCKET,
"staging",
&FileInfo::default(),
"bucket",
"object",
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), entered_rx)
.await
.expect("disk identity preflight starts")
.expect("preflight hook");
rename.abort();
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
let key = disk.get_object_path("bucket", "object").expect("object path");
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
release_tx.send(()).expect("release disk identity preflight");
if !released_before_io {
drop(
tokio::time::timeout(Duration::from_secs(10), successor)
.await
.expect("preflight finishes"),
);
}
assert!(
released_before_io,
"a cancelled request must not retain the reservation before physical mutation starts"
);
}
#[tokio::test]
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
let dir = tempfile::tempdir().expect("disk directory");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
let token = wrapper
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
.await
.expect("reserve key");
let held = wrapper
.io_admission
.background(&CancellationToken::new())
.await
.expect("background permit");
let fi = FileInfo::default();
let rename = crate::disk::with_background_disk_io(
CancellationToken::new(),
wrapper.rename_data_observed_with_guards(
RUSTFS_META_TMP_BUCKET,
"staging",
&fi,
"bucket",
"object",
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
),
);
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
drop(held);
let observation = result.expect("reserved mutation must not wait while holding the object lease");
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
assert!(observation.rejected_before_publication());
}
#[tokio::test]
async fn delete_versions_counts_faulty_drive_availability_rejection() {
let dir = tempfile::tempdir().expect("temp dir should be created");
@@ -6086,67 +6318,6 @@ mod tests {
assert!(health.offline_duration().is_none());
}
#[tokio::test]
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
let directories = [
tempfile::tempdir().expect("faulted drive"),
tempfile::tempdir().expect("healthy drive"),
];
let mut wrappers = Vec::new();
for directory in &directories {
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
wrappers.push(LocalDiskWrapper::new(disk, false));
}
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
Arc::clone(&wrappers[0].disk),
false,
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
);
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
tokio::time::pause();
let operation = || async {
tokio::time::sleep(Duration::from_secs(2)).await;
Ok(())
};
let (faulted, healthy) = tokio::join!(
reconnected.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
wrappers[1].track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_secs(3),
TimeoutHealthAction::IgnoreFailure,
),
);
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
healthy.expect("healthy drive keeps its caller's deadline");
let shorter_started = tokio::time::Instant::now();
let shorter = wrappers[0]
.track_disk_health_with_op_and_timeout_action(
"read_metadata",
operation,
Duration::from_millis(500),
TimeoutHealthAction::IgnoreFailure,
)
.await;
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
assert!(shorter_started.elapsed() < Duration::from_secs(1));
wrappers[0]
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
.await
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
drop(scope);
reconnected
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
.await
.expect("dropping the scope restores reconnected callers' deadlines");
}
#[tokio::test]
async fn ignored_timeout_does_not_mark_drive_failure() {
let dir = tempfile::tempdir().expect("temp dir should be created");
+2 -8
View File
@@ -5,7 +5,6 @@ use super::{FdCache, FdCacheEntry, FdKey};
use std::fs::File;
use std::os::unix::fs::FileExt;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
FdKey {
@@ -53,11 +52,7 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
let (_directory, file) = fixture();
let cache = FdCache::new();
let entry = Arc::new(FdCacheEntry {
file,
len: 3,
invalidated: AtomicBool::new(false),
});
let entry = Arc::new(FdCacheEntry { file, len: 3 });
let old_generation = cache.generation();
cache.invalidate_exact("v", "part").await;
let current_generation = cache.generation();
@@ -130,7 +125,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
assert!(
cache.get(&key("v", "part", direct)).await.is_some(),
"new entries must survive the earlier volume invalidation"
"new entries must survive the earlier volume predicate"
);
}
cache.clear();
@@ -169,7 +164,6 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
let replacement = Arc::new(FdCacheEntry {
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
len: 11,
invalidated: AtomicBool::new(false),
});
for direct in [false, true] {
let key = key("v", "part", direct);
File diff suppressed because it is too large Load Diff
+70 -3
View File
@@ -444,12 +444,29 @@ async fn prepare_rename_commit_and_destination_metadata(
#[derive(Debug)]
pub(in crate::disk) struct LocalRenamePreflightRejection(());
impl LocalRenamePreflightRejection {
pub(in crate::disk) fn not_dispatched() -> Self {
Self(())
}
}
#[derive(Default)]
pub(super) struct RenameDataState {
namespace_owner: Option<Arc<dyn Send + Sync>>,
metadata_reservation: Option<os::MetadataMutationReservation>,
preflight_rejection: Option<LocalRenamePreflightRejection>,
}
#[cfg(test)]
impl RenameDataState {
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
Self {
metadata_reservation: Some(reservation),
..Default::default()
}
}
}
// Keep optional native MPU state machines out of ordinary rename frames.
// Construct the future here as well as pinning it, so its temporary stays off
// the caller's poll stack. Call only from the existing native MPU branches.
@@ -491,6 +508,15 @@ impl LocalDisk {
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
}
let mut fi = fi;
// Pre-acquired metadata must never reverse quota or multipart-source
// lock order. These paths need their own reservation protocol.
if state.metadata_reservation.is_some()
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
{
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
@@ -586,7 +612,27 @@ impl LocalDisk {
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
None => state.namespace_owner.take(),
};
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
// A reserved lease is single-use and bound to the exact disk path.
// Keep the independent namespace/source owner alongside it; a pre-held
// mutex cannot be acquired again without deadlocking.
let reservation_fence = state
.metadata_reservation
.as_ref()
.and_then(|reservation| reservation.fence());
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
if multipart_staging.is_some() || quota_fence_claim.is_some() {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
let lease = reservation
.into_lease_for(&root_destination_object_path, self.reservation_instance)
.inspect_err(|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
})?;
Arc::new((lease, metadata_owner))
} else {
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
};
drop(metadata_wait);
#[cfg(unix)]
let parent_receipt = if native_timing {
@@ -600,17 +646,23 @@ impl LocalDisk {
};
#[cfg(unix)]
let mut receipt_mutation = native_timing
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
&self.root,
dst_volume,
&root_destination_object_path,
staging_prefix.as_deref(),
Some(metadata_lease),
Some(metadata_owner),
)
.await;
drop(namespace_wait);
// The volume read lease prevents deletion of this disk's bucket until
// publication finishes. Recheck after its wait, before any mutation.
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
if let Some(claim) = quota_fence_claim {
mutation_lease.attach_external_guard(claim);
}
@@ -2549,8 +2601,23 @@ impl LocalDisk {
dst_path: &str,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> super::super::RenameDataObservation {
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
.await
}
pub(in crate::disk) async fn rename_data_observed_with_reservation(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
) -> super::super::RenameDataObservation {
let (namespace_owner, metadata_reservation) = owners;
let mut state = RenameDataState {
namespace_owner,
metadata_reservation,
..Default::default()
};
let result = self
+112 -12
View File
@@ -106,6 +106,8 @@ pub(crate) struct RenameDataGuards {
pub(crate) scanner_publication_lease_token: Option<Uuid>,
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
pub(crate) metadata_reservation_token: Option<Uuid>,
}
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
@@ -122,6 +124,13 @@ impl RenameDataObservation {
}
}
fn not_dispatched(error: DiskError) -> Self {
Self {
result: Err(error),
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
}
}
pub(crate) fn rejected_before_publication(&self) -> bool {
self.result.is_err() && self.preflight_rejection.is_some()
}
@@ -311,6 +320,82 @@ pub enum Disk {
}
impl Disk {
/// Prepare an inert single-use reservation on this disk.
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
pub async fn prepare_object_metadata(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> Result<Uuid> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.prepare_object_metadata(bucket, object),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
}
}
/// Refresh an inert token while its sender stages the write.
pub async fn renew_object_metadata(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: std::time::Duration,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.renew_object_metadata(bucket, object, token),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await,
}
}
/// Enqueue a prepared token in object mutation order.
pub async fn activate_object_metadata(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: std::time::Duration,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.activate_object_metadata(bucket, object, token),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
}
}
/// Cancel an unclaimed reservation. False does not prove that a claim has
/// not already started a physical mutation.
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
match self {
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
}
}
/// Physical ownership belongs to the node hosting the local attachment.
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
match self {
@@ -1251,18 +1336,33 @@ impl Disk {
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
.await
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
guards.scanner_publication_lease_token,
)
.await,
),
Disk::Remote(remote_disk) => {
if guards.metadata_reservation.is_some() {
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
}
if let Some(token) = guards.metadata_reservation_token {
if guards.scanner_publication_lease_token.is_some() {
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
}
return RenameDataObservation::unknown(
remote_disk
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
.await,
);
}
RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
guards.scanner_publication_lease_token,
)
.await,
)
}
}
}
File diff suppressed because it is too large Load Diff
+38 -1
View File
@@ -28,6 +28,7 @@ use tokio_util::sync::CancellationToken;
#[derive(Clone)]
pub struct NamespaceLockFence {
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
local_probes: Arc<Vec<rustfs_lock::fast_lock::guard::FastLockGuardProbe>>,
#[cfg(test)]
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
}
@@ -44,6 +45,7 @@ impl NamespaceLockFence {
pub(crate) fn new() -> Self {
Self {
signals: Arc::default(),
local_probes: Arc::default(),
#[cfg(test)]
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
}
@@ -58,17 +60,28 @@ impl NamespaceLockFence {
{
return true;
}
self.signals.iter().any(|signal| signal.is_lost())
self.signals.iter().any(|signal| signal.is_lost()) || self.local_probes.iter().any(|probe| probe.is_lost())
}
pub(crate) fn add_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
Arc::make_mut(&mut self.signals).push(signal);
}
fn add_local_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
if let rustfs_lock::NamespaceLockGuard::Fast(guard) = guard
&& let Some(probe) = guard.release_probe()
{
Arc::make_mut(&mut self.local_probes).push(probe);
}
}
fn extend(&mut self, other: &Self) {
if !Arc::ptr_eq(&self.signals, &other.signals) {
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
}
if !Arc::ptr_eq(&self.local_probes, &other.local_probes) {
Arc::make_mut(&mut self.local_probes).extend(other.local_probes.iter().cloned());
}
#[cfg(test)]
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
@@ -106,6 +119,7 @@ impl NamespaceLockSignalTestFence {
) -> Self {
let fence = NamespaceLockFence {
signals: Arc::default(),
local_probes: Arc::default(),
forced_lost: Arc::new(vec![loss_handle]),
};
let signal_key = Arc::as_ptr(signal) as usize;
@@ -160,6 +174,20 @@ pub struct ObjectLockConfigSnapshot {
}
impl ObjectLockConfigSnapshot {
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
self._lifecycle_guard.is_some()
}
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
self.bucket.as_deref() == Some(bucket)
&& incarnation.is_some()
&& self.bucket_incarnation_id == incarnation
&& self
._lifecycle_guard
.as_ref()
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
}
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
Self {
store_id: None,
@@ -208,6 +236,7 @@ impl ObjectLockConfigSnapshot {
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
lifecycle_fence.add_signal(signal);
}
lifecycle_fence.add_local_guard(&lifecycle_guard);
Self {
store_id: Some(store_id),
bucket: Some(bucket.to_string()),
@@ -1353,6 +1382,13 @@ impl ObjectOptions {
if let Some(signal) = guard.lock_lost_signal() {
self.add_namespace_lock_lost_signal(signal);
}
if let rustfs_lock::NamespaceLockGuard::Fast(local) = guard
&& !local.is_disabled()
{
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.add_local_guard(guard);
}
}
pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
@@ -1360,6 +1396,7 @@ impl ObjectOptions {
if let Some(signal) = guard.lock_lost_signal() {
fence.add_signal(signal);
}
fence.add_local_guard(guard);
}
pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
+39 -1
View File
@@ -58,7 +58,7 @@ use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -71,6 +71,9 @@ const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
// Bound abandoned speculative RPCs independently for each storage instance.
pub(crate) const SPECULATIVE_RESERVATION_LIMIT: usize = 64;
/// A lease is deliberately short-lived. The coordinator treats expiry as a
/// failed publication rather than silently continuing with a peer that may
/// have started movement after the lease was abandoned.
@@ -101,6 +104,12 @@ pub(crate) struct NamespaceCommitGuard {
counted: bool,
}
impl NamespaceCommitGuard {
pub(crate) fn covers_instance(&self, ctx: &Arc<InstanceContext>) -> bool {
Arc::ptr_eq(&self.ctx, ctx)
}
}
impl Drop for NamespaceCommitGuard {
fn drop(&mut self) {
if self.counted {
@@ -332,6 +341,7 @@ pub struct InstanceContext {
scanner_publication_state: AtomicU8,
namespace_commits: AtomicU64,
namespace_commit_generation: AtomicU64,
speculative_reservations: Arc<Semaphore>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
delete_recovery_hints: DeleteRecoveryHints,
@@ -392,6 +402,7 @@ impl InstanceContext {
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
namespace_commits: AtomicU64::new(0),
namespace_commit_generation: AtomicU64::new(0),
speculative_reservations: Arc::new(Semaphore::new(SPECULATIVE_RESERVATION_LIMIT)),
object_encryption_resolver: OnceLock::new(),
delete_recovery_hints: DeleteRecoveryHints::default(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -413,6 +424,11 @@ impl InstanceContext {
self.lock_manager.clone()
}
/// Skip speculative work when its detached owners already fill the instance budget.
pub(crate) fn try_acquire_speculative_reservation(&self) -> Option<OwnedSemaphorePermit> {
Arc::clone(&self.speculative_reservations).try_acquire_owned().ok()
}
/// Run a multi-step mutation on its own task and wait for its output.
///
/// Dropping the returned future leaves the task running to completion, so
@@ -1143,6 +1159,26 @@ mod tests {
assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false));
}
#[test]
fn speculative_reservation_budget_is_bounded_and_instance_local() {
let first = InstanceContext::new();
let other = InstanceContext::new();
let mut permits = (0..super::SPECULATIVE_RESERVATION_LIMIT)
.map(|_| first.try_acquire_speculative_reservation().expect("initial permit"))
.collect::<Vec<_>>();
assert!(first.try_acquire_speculative_reservation().is_none());
assert!(
other.try_acquire_speculative_reservation().is_some(),
"budgets belong to their own instance"
);
drop(permits.pop());
let replacement = first
.try_acquire_speculative_reservation()
.expect("dropping an owner frees its slot");
assert!(first.try_acquire_speculative_reservation().is_none());
drop(replacement);
}
#[test]
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
let first = Arc::new(InstanceContext::new());
@@ -1151,6 +1187,8 @@ mod tests {
other.set_scanner_publication_state(false);
assert!(first.scanner_publication_state_allowed());
let one = first.begin_namespace_commit();
assert!(one.covers_instance(&first));
assert!(!one.covers_instance(&other));
let shared_owner = Arc::clone(&one);
let two = first.begin_namespace_commit();
assert!(first.namespace_commits_pending());
+3 -106
View File
@@ -318,21 +318,6 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
}
}
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
match rebalance_error_source(err) {
Error::Lock(err) => is_rebalance_transient_lock_error(err),
// No metadata was written when lock admission failed. Each attempt
// must acquire its quorum again before loading or saving this run.
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket,
object,
..
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
_ => false,
}
}
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
if err.kind() == std::io::ErrorKind::TimedOut {
return true;
@@ -400,7 +385,9 @@ where
match result {
Ok(value) => return Ok(value),
Err(err) => {
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
if attempt.saturating_add(1) >= max_attempts.max(1)
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
{
return Err(err);
}
#[cfg(test)]
@@ -732,82 +719,6 @@ mod error_source_tests {
}
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
let mut attempts = 0;
retry_rebalance_metadata_access(None, 3, || {
attempts += 1;
std::future::ready(if attempts == 1 {
resolve_rebalance_meta_save_result(
Err(rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)),
"stop_rebalance",
)
} else {
Ok(())
})
})
.await
.expect("a later quorum-protected admission must be allowed to succeed");
assert_eq!(attempts, 2);
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
for max_attempts in [0, 1, 3] {
let mut attempts = 0;
let result = retry_rebalance_metadata_access(None, max_attempts, || {
attempts += 1;
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)))
})
.await;
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
assert_eq!(attempts, max_attempts.max(1));
assert!(matches!(
rebalance_error_source(&error),
Error::NamespaceLockQuorumUnavailable {
mode: "write",
required: 1,
achieved: 0,
..
}
));
}
}
#[tokio::test(start_paused = true)]
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
let mut attempts = 0;
let result = retry_rebalance_metadata_access(None, 3, || {
attempts += 1;
std::future::ready(Err::<(), _>(if attempts == 1 {
rebalance_meta_lock_error(
rustfs_lock::LockError::QuorumNotReached {
required: 1,
achieved: 0,
},
"write",
)
} else {
Error::OperationCanceled
}))
})
.await;
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
assert!(matches!(result, Err(Error::OperationCanceled)));
}
#[tokio::test]
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
for err in [
@@ -820,20 +731,6 @@ mod error_source_tests {
required: 3,
achieved: 2,
},
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: "data-bucket".to_string(),
object: REBAL_META_NAME.to_string(),
required: 1,
achieved: 0,
},
Error::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
object: "other-metadata.bin".to_string(),
required: 1,
achieved: 0,
},
Error::other("stale rebalance run rejected: lock acquisition timed out"),
Error::other("rebalance distributed run fence lost"),
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
@@ -2538,6 +2538,8 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
pub(in crate::set_disk) committed_file_info: FileInfo,
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
pub(in crate::set_disk) tail_has_owned_bucket_fences: bool,
}
pub(in crate::set_disk) struct RenameTailCleanup {
@@ -3468,6 +3470,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
rollback_receipt: Option<RenameRollbackReceipt>,
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
}
impl<'a> RenameDataFenceOptions<'a> {
@@ -3483,6 +3487,8 @@ impl<'a> RenameDataFenceOptions<'a> {
rollback_receipt: None,
multipart_proof_authority: None,
multipart_commit_owner: None,
reservation_ctx: None,
prepared_reservations: None,
}
}
@@ -3519,6 +3525,415 @@ impl<'a> RenameDataFenceOptions<'a> {
self.namespace_commit_guard = namespace_commit_guard;
self
}
pub(in crate::set_disk) fn with_reservation_context(
mut self,
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
) -> Self {
self.reservation_ctx = ctx;
self.prepared_reservations = reservations;
self
}
}
#[derive(Clone)]
struct MetadataRenewalTarget {
disk: DiskStore,
token: Uuid,
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
}
struct MetadataRenewalGroup {
targets: Arc<parking_lot::Mutex<HashMap<usize, MetadataRenewalTarget>>>,
task: tokio::task::JoinHandle<()>,
}
impl Drop for MetadataRenewalGroup {
fn drop(&mut self) {
self.task.abort();
}
}
pub(crate) struct RenameMetadataReservation {
disk: DiskStore,
token: Option<Uuid>,
// Remote reservation protocol v2 and the local owned path both retain
// bucket publication and lifecycle guards until the physical write drains.
owns_bucket_fence: bool,
short: bool,
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
renewal: Option<(Arc<MetadataRenewalGroup>, usize)>,
}
impl RenameMetadataReservation {
fn stop_renewal(&mut self) {
if let Some((group, index)) = self.renewal.take() {
let mut targets = group.targets.lock();
targets.remove(&index);
if targets.is_empty() {
group.task.abort();
}
}
}
async fn cancel(&mut self) -> disk::error::Result<()> {
self.stop_renewal();
let Some(token) = self.token else { return Ok(()) };
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
Ok(Ok(true)) => {
self.token = None;
Ok(())
}
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
Ok(Err(error)) => Err(error),
Err(_) => Err(DiskError::Timeout),
}
}
}
impl Drop for RenameMetadataReservation {
fn drop(&mut self) {
self.stop_renewal();
let Some(token) = self.token.take() else { return };
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
let disk = self.disk.clone();
runtime.spawn(async move {
let _ = disk.cancel_object_metadata_reservation(token).await;
});
}
}
}
#[derive(Clone, Copy)]
pub(crate) enum RenameReservationMode {
Ownerless,
Owned,
ShortOwned,
}
pub(crate) async fn prepare_rename_metadata_reservations(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
incarnation: Uuid,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
owns_bucket_fence: bool,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
prepare_rename_metadata_reservations_until_abandoned(
disks,
bucket,
object,
incarnation,
ctx,
if owns_bucket_fence {
RenameReservationMode::Owned
} else {
RenameReservationMode::Ownerless
},
std::future::pending(),
)
.await
}
pub(crate) async fn prepare_rename_metadata_reservations_until_abandoned(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
incarnation: Uuid,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
mode: RenameReservationMode,
abandoned: impl std::future::Future<Output = ()>,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
let owns_bucket_fence = !matches!(mode, RenameReservationMode::Ownerless);
let short = matches!(mode, RenameReservationMode::ShortOwned);
let mut abandoned = Box::pin(abandoned);
let discarded = Arc::new(std::sync::atomic::AtomicBool::new(false));
let mut pending: FuturesUnordered<_> = disks
.iter()
.enumerate()
.map(|(_index, disk)| {
let discarded = Arc::clone(&discarded);
async move {
let result: disk::error::Result<Option<RenameMetadataReservation>> = async {
if discarded.load(std::sync::atomic::Ordering::Acquire) {
return Ok(None);
}
let Some(disk) = disk else { return Ok(None) };
#[cfg(test)]
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_BEFORE).await;
if disk.is_local() {
crate::store::admit_local_disk(ctx, disk, None, false).await?;
}
if discarded.load(std::sync::atomic::Ordering::Acquire) {
return Ok(None);
}
let prepared_at = tokio::time::Instant::now();
let token = if disk.is_local() && owns_bucket_fence {
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
.await
.ok_or(DiskError::DiskOngoingReq)?;
if short {
store
.reserve_local_object_metadata_short(
&disk.endpoint().to_string(),
bucket,
object,
incarnation,
Duration::from_millis(50),
)
.await?
} else {
store
.reserve_local_object_metadata_at_incarnation(
&disk.endpoint().to_string(),
bucket,
object,
incarnation,
Duration::from_secs(5),
)
.await?
}
} else if short {
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
return Err(DiskError::InvalidPath);
};
remote
.reserve_object_metadata_short(bucket, object, incarnation, Duration::from_millis(50))
.await?
} else {
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
.await?
};
// No await separates receipt from installing the cancellation owner.
let reservation = RenameMetadataReservation {
disk: disk.clone(),
token: Some(token),
owns_bucket_fence: !disk.is_local() || owns_bucket_fence,
short,
renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)),
renewal: None,
};
if short {
// Install the cancellation owner before extending the short token.
// A dropped renew response still leaves us the token to cancel.
disk.renew_object_metadata(bucket, object, token, Duration::from_millis(50))
.await?;
}
#[cfg(test)]
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_AFTER).await;
Ok(Some(reservation))
}
.await;
(_index, result)
}
})
.collect();
let mut reservations: Vec<Option<RenameMetadataReservation>> = (0..disks.len()).map(|_| None).collect();
let mut discard = false;
while !pending.is_empty() {
tokio::select! {
biased;
_ = &mut abandoned, if !discard => {
discard = true;
discarded.store(true, std::sync::atomic::Ordering::Release);
reservations.clear();
}
result = pending.next() => {
let Some((index, result)) = result else { break };
if discard {
drop(result);
} else {
match result {
Ok(reservation) => reservations[index] = reservation,
Err(_) => {
discard = true;
discarded.store(true, std::sync::atomic::Ordering::Release);
reservations.clear();
}
}
}
}
}
}
if discard {
// Drain all in-flight preparations so late tokens get a cancellation owner.
return Ok(None);
}
let targets = Arc::new(parking_lot::Mutex::new(HashMap::new()));
for (index, reservation) in reservations.iter().enumerate() {
if let Some(reservation) = reservation {
let token = reservation.token.ok_or(DiskError::Unexpected)?;
targets.lock().insert(
index,
MetadataRenewalTarget {
disk: reservation.disk.clone(),
token,
renewed_at: Arc::clone(&reservation.renewed_at),
},
);
}
}
if !targets.lock().is_empty() {
let renewal_targets = Arc::clone(&targets);
let renewal_ctx = Arc::clone(ctx);
let renewal_bucket = bucket.to_owned();
let renewal_object = object.to_owned();
// Short leases remain short even after renewal, so a lost renewal and
// cancellation cannot strand a bucket fence for the legacy lease time.
let task = tokio::spawn(async move {
let interval = if short {
Duration::from_millis(500)
} else {
Duration::from_secs(20)
};
let retry = if short {
Duration::from_millis(250)
} else {
Duration::from_secs(5)
};
let lifetime = if short {
Duration::from_secs(2)
} else {
Duration::from_secs(60)
};
let mut delay = interval;
loop {
tokio::time::sleep(delay).await;
let current: Vec<_> = renewal_targets.lock().values().cloned().collect();
if current.is_empty() {
break;
}
// Once any acknowledgment is too old, the complete fanout can
// no longer be safely activated. Stop retrying stale peers;
// their receiver-side tokens still expire on their own.
if current.iter().any(|target| target.renewed_at.lock().elapsed() >= lifetime) {
break;
}
let results = join_all(current.into_iter().map(|target| {
let ctx = Arc::clone(&renewal_ctx);
let bucket = &renewal_bucket;
let object = &renewal_object;
async move {
let started = tokio::time::Instant::now();
let admitted = !target.disk.is_local()
|| crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok();
if admitted
&& target
.disk
.renew_object_metadata(bucket, object, target.token, retry)
.await
.is_ok()
{
*target.renewed_at.lock() = started;
true
} else {
false
}
}
}))
.await;
delay = if results.into_iter().all(|success| success) {
interval
} else {
retry
};
}
});
let group = Arc::new(MetadataRenewalGroup { targets, task });
for (index, reservation) in reservations.iter_mut().enumerate() {
if let Some(reservation) = reservation {
reservation.renewal = Some((Arc::clone(&group), index));
}
}
}
Ok(Some(reservations))
}
pub(crate) fn align_rename_metadata_reservations(
mut reservations: Vec<Option<RenameMetadataReservation>>,
disks: &[Option<DiskStore>],
) -> Option<Vec<Option<RenameMetadataReservation>>> {
if reservations.len() != disks.len() {
return None;
}
let mut aligned = Vec::with_capacity(disks.len());
for disk in disks {
let Some(disk) = disk else {
aligned.push(None);
continue;
};
let index = reservations
.iter()
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
aligned.push(reservations[index].take());
}
// Disks lost while staging need no activation; dropping their still-inert
// reservations schedules cancellation without blocking the commit.
Some(aligned)
}
async fn activate_rename_metadata_reservations(
mut reservations: Vec<Option<RenameMetadataReservation>>,
bucket: &str,
object: &str,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
// Short tokens continue renewing during activation; an activated token
// can otherwise expire while another disk's activation is still pending.
// Long tokens retain their existing non-renewable queued behavior.
for reservation in reservations.iter_mut().flatten() {
if !reservation.short {
reservation.stop_renewal();
}
}
if reservations.iter().flatten().any(|reservation| {
let max_age = if reservation.short {
Duration::from_secs(1)
} else {
Duration::from_secs(50)
};
reservation.renewed_at.lock().elapsed() >= max_age
}) {
return Ok(None);
}
#[cfg(test)]
let test_reservations = &reservations;
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
if reservation.disk.is_local() {
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
}
let token = reservation.token.ok_or(DiskError::Unexpected)?;
let result = reservation
.disk
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
.await;
#[cfg(test)]
if result.is_ok() {
let index = test_reservations
.iter()
.position(|candidate| {
candidate
.as_ref()
.is_some_and(|candidate| std::ptr::eq(candidate, reservation))
})
.expect("activated reservation belongs to the fanout");
rename_fanout_barrier::checkpoint(object, index, rename_fanout_barrier::PHASE_RESERVATION_ACTIVATED).await;
}
result
}))
.await;
for reservation in reservations.iter_mut().flatten() {
reservation.stop_renewal();
}
if activated.into_iter().any(|result| result.is_err()) {
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
if cancelled.into_iter().any(|result| result.is_err()) {
return Err(DiskError::DiskOngoingReq);
}
return Ok(None);
}
Ok(Some(reservations))
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3530,7 +3945,7 @@ type RenameDataLegacyTuple = (
Option<OldCurrentSize>,
);
fn put_rename_early_ack_enabled() -> bool {
pub(crate) fn put_rename_early_ack_enabled() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
}
@@ -3896,6 +4311,8 @@ impl SetDisks {
old_current_source,
committed_file_info,
tail_drain: None,
tail_has_metadata_reservations: false,
tail_has_owned_bucket_fences: false,
})
}
@@ -3992,6 +4409,8 @@ impl SetDisks {
rollback_receipt,
multipart_proof_authority: _,
multipart_commit_owner: _,
reservation_ctx,
prepared_reservations,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4007,6 +4426,21 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
(_, _) => None,
};
let tail_has_metadata_reservations = reservations.is_some();
let tail_has_owned_bucket_fences = reservations.as_ref().is_some_and(|reservations| {
reservations.len() == disks.len()
&& disks.iter().any(Option::is_some)
&& disks.iter().zip(reservations).all(|(disk, reservation)| match disk {
Some(disk) => reservation
.as_ref()
.is_some_and(|owner| owner.owns_bucket_fence && Arc::ptr_eq(&owner.disk, disk)),
None => true,
})
});
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
@@ -4039,6 +4473,11 @@ impl SetDisks {
let file_info = file_info.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let namespace_commit_guard = namespace_commit_guard.clone();
let reservation = reservations
.as_mut()
.and_then(|entries| entries.get_mut(i))
.and_then(Option::take);
let reservation_ctx = reservation_ctx.clone();
tasks.spawn(async move {
let _namespace_commit_guard = namespace_commit_guard.clone();
let mut dispatch_state = RenameDispatchState::NotDispatched;
@@ -4049,6 +4488,17 @@ impl SetDisks {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let disk_admission = if reservation.is_some() && disk.is_local() {
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
// The fanout owner already covers this instance through
// physical completion; only add a disk owner when it does not.
let covered = namespace_commit_guard
.as_ref()
.is_some_and(|guard| guard.covers_instance(ctx));
crate::store::admit_local_disk(ctx, &disk, None, !covered).await?
} else {
None
};
let disk_wait_location_stage = if disk.is_local() {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
} else {
@@ -4074,6 +4524,11 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let _foreground_active_guard = rename_foreground_active_guard(i);
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
} else {
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
};
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
@@ -4083,9 +4538,8 @@ impl SetDisks {
&dst_object,
disk::RenameDataGuards {
scanner_publication_lease_token,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
namespace_owner,
..Default::default()
},
)
@@ -4363,6 +4817,8 @@ impl SetDisks {
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
commit.map(|mut commit| {
commit.tail_drain = Some(tail_drain);
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
commit.tail_has_owned_bucket_fences = tail_has_owned_bucket_fences;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
attach_tail_owner_started,
@@ -4432,6 +4888,8 @@ impl SetDisks {
rollback_receipt,
multipart_proof_authority,
multipart_commit_owner,
reservation_ctx: _,
prepared_reservations: _,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4901,6 +5359,8 @@ impl SetDisks {
old_current_source,
committed_file_info,
tail_drain: None,
tail_has_metadata_reservations: false,
tail_has_owned_bucket_fences: false,
})
}
@@ -7358,6 +7818,9 @@ pub(crate) mod rename_fanout_barrier {
MetadataSlowtailFault,
NonInlineHedgeTimer,
LateMetadataRefresh,
ReservationBefore,
ReservationAfter,
ReservationActivated,
}
impl From<&'static str> for Phase {
@@ -7380,6 +7843,9 @@ pub(crate) mod rename_fanout_barrier {
pub const PHASE_ROLLBACK: Phase = Phase::Rollback;
pub const PHASE_READ_VERSION: Phase = Phase::ReadVersion;
pub const PHASE_METADATA_SLOWTAIL_FAULT: Phase = Phase::MetadataSlowtailFault;
pub const PHASE_RESERVATION_BEFORE: Phase = Phase::ReservationBefore;
pub const PHASE_RESERVATION_AFTER: Phase = Phase::ReservationAfter;
pub const PHASE_RESERVATION_ACTIVATED: Phase = Phase::ReservationActivated;
/// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk.
pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer;
@@ -7454,6 +7920,18 @@ pub(crate) mod rename_fanout_barrier {
}
impl BarrierHandle {
/// Keep the current task parked without intercepting later tasks.
pub fn disarm(&self) {
let mut reg = lock();
if reg
.armed
.get(&self.key)
.is_some_and(|armed| Arc::ptr_eq(&armed.release, &self.release))
{
reg.armed.remove(&self.key);
}
}
/// Await until the armed fan-out task has parked at the checkpoint. Uses a
/// stored-permit `Notify`, so this is race-free regardless of whether the
/// task reaches the checkpoint before or after this call — no sleeps.
@@ -7619,6 +8097,45 @@ mod tests {
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() {
let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await;
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let endpoint = disk.endpoint().to_string();
ctx.local_disk_map()
.write()
.await
.insert(endpoint.clone(), Some(disk.clone()));
let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false)
.await
.expect("prepare")
.expect("complete fanout");
let group = Arc::clone(
&reservations[0]
.as_ref()
.expect("reservation")
.renewal
.as_ref()
.expect("renewal")
.0,
);
ctx.local_disk_map().write().await.remove(&endpoint);
tokio::task::yield_now().await;
tokio::time::pause();
tokio::time::advance(Duration::from_secs(21)).await;
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(41)).await;
for _ in 0..10 {
if group.task.is_finished() {
break;
}
tokio::task::yield_now().await;
}
tokio::time::resume();
assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts");
drop(reservations);
}
#[tokio::test]
async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() {
use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint};
+10
View File
@@ -410,6 +410,10 @@ impl ObjectLockDiagGuard {
self
}
async fn downgrade_committed_tail(&mut self) -> bool {
self.guard.downgrade_committed_tail().await
}
/// Whether the underlying namespace lock's heartbeat has observed a
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
/// point on this so a stale lock holder does not race a double-write.
@@ -902,6 +906,10 @@ mod core;
pub(crate) use core::io_primitives::disk_call_counters;
#[cfg(test)]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
pub(crate) use core::io_primitives::{
RenameMetadataReservation, RenameReservationMode, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
prepare_rename_metadata_reservations_until_abandoned, put_rename_early_ack_enabled,
};
mod ctx;
mod metadata;
mod ops;
@@ -928,6 +936,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(all(test, feature = "test-util"))]
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
#[cfg(test)]
pub(crate) use ops::object::pause_put_object_commit;
#[cfg(test)]
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
#[cfg(any(test, feature = "test-util"))]
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
+493 -9
View File
@@ -34,6 +34,7 @@ mod heal_lock_wait_tests;
#[derive(Clone, Copy)]
struct ExplicitVersionHeal<'a> {
opts: &'a HealOpts,
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
allow_regeneration: bool,
preserve_unrepairable: bool,
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
@@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> {
retirement: Option<&'a MarkerRetirementContext<'a>>,
}
async fn restore_heal_exclusive_guard(
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
) -> disk::error::Result<()> {
tokio::time::timeout(get_lock_acquire_timeout(), async {
loop {
{
let mut guard = guard.lock().await;
if guard.is_lock_lost() {
return Err(DiskError::DiskOngoingReq);
}
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
if !matches!(waiting, Ok(Some(false))) {
return Err(DiskError::Timeout);
}
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
.await
.unwrap_or(false)
{
// A writer may have waited on a peer that upgraded while
// another peer was still waiting for its readers to drain.
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
return if matches!(waiting, Ok(Some(false))) {
Ok(())
} else {
Err(DiskError::Timeout)
};
}
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.map_err(|_| DiskError::Timeout)?
}
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
loop {
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let waiting =
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
if !matches!(waiting, Ok(Some(false))) {
return;
}
}
}
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
drop(tokio::spawn(async move {
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
}));
}
// Keep the large heal state machine and its construction temporary out of
// each dispatch caller's poll frame. Ordinary object writes do not use this
// heal-only allocation boundary.
@@ -844,6 +897,7 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts,
staging_guard: None,
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: None,
@@ -994,6 +1048,7 @@ impl SetDisks {
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
let ExplicitVersionHeal {
opts,
staging_guard,
allow_regeneration: allow_explicit_version_regen,
preserve_unrepairable,
commit_fence,
@@ -1701,7 +1756,29 @@ impl SetDisks {
let dst_data_dir = data_dir;
if !latest_meta.deleted && !latest_meta.is_remote() {
let stage_result: disk::error::Result<Option<DiskError>> = async {
let mut shared_staging = false;
if let Some(guard) = staging_guard {
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
guard.lock().await.supports_heal_lock_transition().await
})
.await
.unwrap_or(false);
if supported {
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
guard.lock().await.downgrade_committed_tail().await
})
.await
.unwrap_or(false);
if converted {
shared_staging = true;
} else {
// Some peers may have converted. Abort without committing;
// dropping the guard releases each retained lease.
return Err(DiskError::Timeout);
}
}
}
let stage = async {
let erasure_info = latest_meta.erasure.clone();
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
@@ -1926,17 +2003,43 @@ impl SetDisks {
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
Ok(None)
}
.await;
};
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
if let Some(guard) = staging_guard {
tokio::select! {
result = stage => result,
_ = wait_for_heal_writer(guard) => {
// The UUID prefix is private to this attempt. Cleanup must not
// retain the namespace lock while a foreground writer waits;
// the stale-tmp sweeper also covers a cancelled disk operation.
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
return Err(DiskError::Timeout);
}
}
} else {
stage.await
}
} else {
stage.await
};
match stage_result {
Ok(None) => {}
failure => {
// Release readers and writers before removing all parts
// staged under this heal's exclusively owned prefix.
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
if shared_staging {
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
} else {
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
}
return failure.map(|error| (result, error));
}
}
if shared_staging
&& let Some(guard) = staging_guard
&& let Err(error) = restore_heal_exclusive_guard(guard).await
{
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
return Err(error);
}
}
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
if _read_repair_read_lock_guard
@@ -3616,6 +3719,7 @@ impl SetDisks {
&version,
ExplicitVersionHeal {
opts: &opts,
staging_guard: None,
allow_regeneration: false,
preserve_unrepairable: true,
commit_fence: Some(can_commit),
@@ -3676,6 +3780,7 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &read_opts,
staging_guard: None,
allow_regeneration: false,
preserve_unrepairable: true,
commit_fence: Some(&can_commit),
@@ -3702,7 +3807,7 @@ impl SetDisks {
.new_ns_lock(bucket, object)
.await
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
Some(Arc::new(
Some(Arc::new(tokio::sync::Mutex::new(
measure_heal_object_lock(
"heal_object",
HealObjectLockKind::Write,
@@ -3714,7 +3819,7 @@ impl SetDisks {
.narrow_to_disk()
.unwrap_or_else(DiskError::other)
})?,
))
)))
} else {
None
};
@@ -3732,10 +3837,18 @@ impl SetDisks {
} else {
None
};
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
guard.lock().await.lock_lost_signal()
} else {
None
};
let locks_lost = || {
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
};
// Recovery cleanup can outlive its future. Never permit reads until
// every physical operation holding its recovery owner has finished.
let recovery_tail = Arc::new(());
let mut recovery_removed_version = false;
if !opts.read_repair
&& _write_lock_guard.is_some()
@@ -3762,6 +3875,7 @@ impl SetDisks {
Arc::new((
scope.clone(),
self.ctx.begin_namespace_commit(),
Arc::clone(&recovery_tail),
_write_lock_guard.clone(),
transaction_guard.clone(),
)) as Arc<dyn Send + Sync>
@@ -3885,6 +3999,9 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &inner_opts,
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
.then_some(_write_lock_guard.as_ref())
.flatten(),
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&commit_fence),
@@ -3909,6 +4026,9 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &inner_opts,
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
.then_some(_write_lock_guard.as_ref())
.flatten(),
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&commit_fence),
@@ -6047,6 +6167,7 @@ mod heal_result_report_tests {
"",
ExplicitVersionHeal {
opts: &opts,
staging_guard: None,
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&can_commit),
@@ -10637,6 +10758,369 @@ mod heal_result_report_tests {
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-storage-entry";
let object = "object.bin";
let expected = vec![0x37; 1024 * 1024];
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(expected.clone()),
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write object");
let metadata = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external object data").to_string())
.join("part.1");
tokio::fs::remove_file(&missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal = tokio::spawn(async move {
let mut absence = None;
let result = heal_set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
read_repair: false,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await;
(result, absence)
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("read-repair reached shard reconstruction");
let mut reader = tokio::time::timeout(
Duration::from_millis(200),
set.get_object_reader(
bucket,
object,
None,
http::HeaderMap::new(),
&ObjectOptions {
suppress_read_repair: true,
..Default::default()
},
),
)
.await
.expect("background heal reconstruction should allow concurrent readers")
.expect("open object while reconstruction is paused");
let mut during_heal = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
.await
.expect("read completes while reconstruction is paused")
.expect("read object during reconstruction");
assert_eq!(during_heal, expected);
drop(reader);
release.notify_one();
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("repair finishes")
.expect("repair task joins");
let (_, error) = result.expect("repair operation succeeds");
assert!(error.is_none(), "repair failed: {error:?}");
assert!(absence.is_none());
assert!(missing.exists(), "missing shard was not restored");
let mut reader = set
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read repaired object");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read full object");
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-long-put";
let object = "object.bin";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original object");
let metadata = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external data").to_string())
.join("part.1");
tokio::fs::remove_file(missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal = tokio::spawn(async move {
let mut absence = None;
heal_set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("reconstruction started");
let put = tokio::time::timeout(
Duration::from_secs(7),
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
)
.await;
release.notify_one();
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("heal finishes after releasing reconstruction")
.expect("heal task joins");
put.expect("foreground PUT finishes during reconstruction")
.expect("foreground PUT must not time out");
assert!(
matches!(heal_result, Err(Error::Timeout)),
"interrupted heal must be retryable: {heal_result:?}"
);
drop(_pause);
let mut absence = None;
let (_, retry_error) = set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
.expect("a later heal must retry the current object");
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
let mut reader = set
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read foreground write after heal");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read full object");
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-version-delete";
let object = "object.bin";
set.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let original = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
&ObjectOptions {
versioned: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original version");
let version = original.version_id.expect("version id").to_string();
let survivor = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x52; 1024]),
&ObjectOptions {
versioned: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write surviving version")
.version_id
.expect("surviving version id");
let metadata = disks[0]
.read_version("", bucket, object, &version, &ReadOptions::default())
.await
.expect("read original metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external data").to_string())
.join("part.1");
tokio::fs::remove_file(missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal_version = version.clone();
let heal = tokio::spawn(async move {
heal_set
.heal_object_with_retirement(
bucket,
object,
&heal_version,
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut None,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("reconstruction started");
let delete = tokio::time::timeout(
Duration::from_secs(7),
set.delete_object(
bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
),
)
.await;
release.notify_one();
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("heal finishes")
.expect("heal task joins");
delete
.expect("foreground delete finishes")
.expect("foreground delete succeeds");
assert!(
matches!(heal_result, Err(Error::Timeout)),
"interrupted heal must be retryable: {heal_result:?}"
);
let result = set
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
..Default::default()
},
)
.await;
assert!(
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
"stale heal must not resurrect the deleted version: {result:?}"
);
let mut survivor = set
.get_object_reader(
bucket,
object,
None,
http::HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(survivor.to_string()),
..Default::default()
},
)
.await
.expect("surviving version remains readable");
let mut body = Vec::new();
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
assert_eq!(body, vec![0x52; 1024]);
for disk in &disks {
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
assert!(
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"deleted version must remain absent on each disk: {physical:?}"
);
}
}
#[tokio::test]
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
+26 -26
View File
@@ -3266,19 +3266,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let mut decommission_target_lock_covered = false;
let mut decommission_capacity_guard = None;
if opts.http_preconditions.is_some() {
if write_context.acquires_namespace() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_precondition", bucket, object)
.await?,
);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
#[cfg(test)]
{
@@ -3290,7 +3277,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
self.pool_index,
bucket,
object,
!write_context.acquires_namespace() || object_lock_guard.is_some(),
!write_context.acquires_namespace(),
false,
DecommissionCapacityAdmission::ExistingMultipart,
)
.await?;
@@ -3304,6 +3292,28 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.await?,
);
}
// Evaluate the precondition after taking the fixed decommission domain
// and the target domain, so a staged upload on the retiring source
// cannot publish while migration holds its source read anchor.
if opts.http_preconditions.is_some() {
// The store's earlier cross-pool check is advisory. A write in a
// different pool can complete before this fixed-domain guard is
// acquired, so recheck all pools while it excludes those writes.
if decommission_object_lock_guard.is_some()
&& !opts.data_movement
&& !opts.replication_request
&& !opts.no_lock
&& !opts.skip_decommissioned
&& !opts.skip_rebalancing
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
&& let Some(err) = store.check_write_precondition(bucket, object, opts, true).await
{
return Err(err);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
let upload_guard = self
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
.await?;
@@ -10060,7 +10070,7 @@ mod tests {
// Real disk callbacks and health probes must share wall-clock deadlines.
// Advancing Tokio time can expire unrelated healthy-drive probes.
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "mpu-quorum-success-physical";
@@ -10087,8 +10097,6 @@ mod tests {
local.observe_multipart_cleanup_for_test(&upload_path)
}).collect::<Vec<_>>();
let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
// Fault only the held leg; healthy disks keep their normal I/O budgets.
let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
@@ -10245,15 +10253,7 @@ mod tests {
let size = copy_info.get_actual_size().expect("actual source size");
copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
.await.expect("cross-key Copy cannot wait on the old successful logical owner")
.unwrap_or_else(|error| {
let health = disks.iter().enumerate().map(|(slot, disk)| {
let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
let metrics = local.metrics_snapshot();
(slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
}).collect::<Vec<_>>();
panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
});
.await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
drop(copy_info);
let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
let mut actual = Vec::new();
+484 -13
View File
@@ -4184,7 +4184,19 @@ impl SetDisks {
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
.await
}
pub(crate) async fn put_object_with_prepared_reservations(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
.await
}
@@ -4203,7 +4215,7 @@ impl SetDisks {
// The owned publication capability supplies the exact target write
// domain at commit time; the generic PUT path must not reacquire it.
opts.no_lock = true;
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
.await
}
@@ -4214,8 +4226,9 @@ impl SetDisks {
data: &'a mut PutObjReader,
opts: &'a ObjectOptions,
publication_fence: Option<RemoteTuplePublicationFence>,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
}
async fn put_object_with_old_current_size_inner(
@@ -4225,6 +4238,7 @@ impl SetDisks {
data: &mut PutObjReader,
opts: &ObjectOptions,
mut publication_fence: Option<RemoteTuplePublicationFence>,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
let protect_write = opts.shard_integrity_write_enabled();
@@ -4789,6 +4803,86 @@ impl SetDisks {
.await?;
}
// A new version has no outer bucket snapshot. Prepare its remote
// owners after staging, before taking lifecycle or object locks.
let prepare_new_version = reservations.is_none()
&& !opts.overwrites_existing_version()
&& !self.ctx.lock_manager().is_disabled()
&& crate::set_disk::put_rename_early_ack_enabled()
&& write_context.acquires_namespace()
&& write_context.allows_early_ack(opts.write_completion)
&& publication_fence.is_none()
&& opts.scanner_publication_commit_scope.is_none()
&& opts.decommission_capacity_admission.is_none()
&& opts.decommission_capacity.is_none()
&& opts.quota_admission.is_none()
&& opts.object_lock_config_snapshot.is_none()
&& opts.bucket_lifecycle_lock_fence.is_none()
&& opts.namespace_lock_fence.is_none()
&& !opts.data_movement
&& !is_meta_bucketname(bucket)
&& opts
.put_object_cancellation
.as_ref()
.is_none_or(|token| !token.is_cancelled())
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
.await
.is_some();
#[cfg(any(test, feature = "test-util"))]
if prepare_new_version {
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
}
// Destructive writes prepare before the bucket snapshot. Local-only
// fixtures without a metadata store can prepare before the object lock.
let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
&& crate::set_disk::put_rename_early_ack_enabled()
&& write_context.acquires_namespace()
&& write_context.allows_early_ack(opts.write_completion)
&& publication_fence.is_none()
&& opts.scanner_publication_commit_scope.is_none()
&& (opts.decommission_capacity_admission.is_none() || reservations.is_some())
&& !opts.data_movement
&& !is_meta_bucketname(bucket)
{
if let Some(prepared) = reservations {
crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
} else if prepare_new_version && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil()) {
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
&shuffle_disks,
bucket,
object,
incarnation,
&self.ctx,
true,
)
.await?
} else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
.await
.is_none()
&& let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
{
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
&shuffle_disks,
bucket,
object,
incarnation,
&self.ctx,
false,
)
.await?
} else {
None
}
} else {
None
};
// Conditional writes need the exclusive fixed domain for an
// atomic cross-pool precondition check at publication time.
let ordered_put = prepared_metadata_reservations.is_some() && opts.http_preconditions.is_none();
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
#[cfg(test)]
{
@@ -4801,6 +4895,7 @@ impl SetDisks {
bucket,
object,
!write_context.acquires_namespace() || object_lock_guard.is_some(),
ordered_put,
DecommissionCapacityAdmission::Mutation,
)
.await?;
@@ -4812,7 +4907,9 @@ impl SetDisks {
|| (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered)
{
#[cfg(any(test, feature = "test-util"))]
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
if !prepare_new_version {
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
}
if let Some(expected_incarnation_id) = source_bucket_incarnation_id
&& opts.bucket_lifecycle_lock_fence.is_none()
{
@@ -4847,10 +4944,21 @@ impl SetDisks {
let namespace_lock_acquired_to_commit_ready_started = rustfs_io_metrics::put_stage_timer();
let commit_ready_precondition_and_timestamp_started = rustfs_io_metrics::put_stage_timer();
if opts.http_preconditions.is_some()
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
{
return Err(err);
if opts.http_preconditions.is_some() {
if decommission_object_lock_guard.is_some()
&& !opts.data_movement
&& !opts.replication_request
&& !opts.no_lock
&& !opts.skip_decommissioned
&& !opts.skip_rebalancing
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
&& let Some(err) = store.check_write_precondition(bucket, object, opts, true).await
{
return Err(err);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
if bucket == RUSTFS_META_BUCKET
@@ -5268,6 +5376,11 @@ impl SetDisks {
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
let commit_object_lock_snapshot = opts
.object_lock_config_snapshot
.as_ref()
.filter(|snapshot| snapshot.owns_lifecycle_guard())
.cloned();
// A scanner publication scope owns the movement permit until the
// complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases
@@ -5281,6 +5394,26 @@ impl SetDisks {
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& commit_owns_namespace_guard
&& commit_scanner_publication_scope.is_none();
let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
|| commit_object_lock_snapshot
.as_ref()
.is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
let reservation_ctx = (commit_allows_early_ack
&& write_context.acquires_namespace()
&& commit_object_lock_guard.is_some()
&& (commit_decommission_object_lock_guard.is_none() || ordered_put)
&& (commit_decommission_capacity_guard.is_none() || prepared_metadata_reservations.is_some())
&& commit_publication_guard.is_none()
&& !quota_mutation_fence
&& !opts.data_movement
&& !is_meta_bucketname(bucket)
&& lifecycle_owned)
.then(|| Arc::clone(&self.ctx));
let prepared_metadata_reservations = if reservation_ctx.is_some() {
prepared_metadata_reservations
} else {
None
};
// Full-tail callers also transfer owned guards to the coordinator:
// cancelling their ACK waiter must not cancel an in-flight rename.
let detach_commit_owner = commit_scanner_publication_scope.is_some()
@@ -5363,7 +5496,7 @@ impl SetDisks {
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _decommission_capacity_guard
.as_ref()
.is_some_and(|guard| guard.is_lock_lost())
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
@@ -5437,7 +5570,7 @@ impl SetDisks {
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _decommission_capacity_guard
.as_ref()
.is_some_and(|guard| guard.is_lock_lost())
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
@@ -5604,6 +5737,7 @@ impl SetDisks {
)
.with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone())
.with_reservation_context(reservation_ctx, prepared_metadata_reservations)
.with_namespace_commit_guard(
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
),
@@ -5627,6 +5761,7 @@ impl SetDisks {
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
}
let mut rename_guard_release = None;
let mut committed_tail_guard = None;
let mut needs_immediate_heal = false;
let mut tail_owns_tmp_cleanup = false;
rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -5658,11 +5793,27 @@ impl SetDisks {
.or_else(|| commit_version_suspended.then(Uuid::nil))
.map(|version_id| version_id.to_string());
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let object_lock_guard = _object_lock_guard.take();
let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
None
} else {
_object_lock_guard
.take()
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
};
committed_tail_guard = object_lock_guard.clone();
let publication_guard = _publication_guard.take();
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
let decommission_capacity_guard = _decommission_capacity_guard.take();
// The quorum committed this snapshot. Receiver-owned
// fences keep its bucket generation protected while
// the remaining ordered disk writes drain.
let object_lock_snapshot = if rename_commit.tail_has_owned_bucket_fences {
drop(commit_object_lock_snapshot);
None
} else {
commit_object_lock_snapshot
};
let cleanup_bucket = commit_bucket.clone();
let cleanup_object = commit_object.clone();
let heal_set = commit_set.clone();
@@ -5683,6 +5834,7 @@ impl SetDisks {
write_context,
publication_guard,
bucket_lifecycle_guard,
object_lock_snapshot,
decommission_object_lock_guard,
decommission_capacity_guard,
),
@@ -5704,6 +5856,7 @@ impl SetDisks {
write_context,
publication_guard,
bucket_lifecycle_guard,
object_lock_snapshot,
decommission_object_lock_guard,
decommission_capacity_guard,
),
@@ -5717,6 +5870,11 @@ impl SetDisks {
drop(write_context);
drop(publication_guard);
drop(bucket_lifecycle_guard);
drop(object_lock_snapshot);
// Every rename has settled; the remaining cleanup
// keeps the capacity fence but no longer publishes
// object metadata.
drop(decommission_object_lock_guard);
cleanup_runtime::run(cleanup_reservation, async move {
let zero_target_cleanup = targets.is_empty();
rustfs_io_metrics::record_put_object_stage_duration(
@@ -5763,7 +5921,6 @@ impl SetDisks {
tmp_delete_started,
);
let guard_drop_started = rustfs_io_metrics::put_stage_timer();
drop(decommission_object_lock_guard);
drop(decommission_capacity_guard);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP,
@@ -5972,6 +6129,13 @@ impl SetDisks {
lock_held_post_rename_started,
);
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
if let Some(guard) = committed_tail_guard.take() {
// The committed tail still owns every physical rename. Convert
// only after metadata invalidation and before acknowledging PUT.
// Unsupported peers keep their exclusive leases and the tail
// retains ownership until its physical work has drained.
let _ = guard.lock().await.downgrade_committed_tail().await;
}
if let Some(release) = rename_guard_release.take() {
let _ = release.send(true);
}
@@ -8071,6 +8235,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PutObjectCommitPause {
#[cfg(test)]
BeforeBucketSnapshot,
BeforeNamespace,
AfterNamespace,
AfterQuotaReservation,
@@ -8165,7 +8331,7 @@ impl Drop for PutObjectCommitBarrier {
}
#[cfg(any(test, feature = "test-util"))]
async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) {
pub(crate) async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) {
let barrier = {
let mut slot = PUT_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
@@ -22181,6 +22347,117 @@ mod put_object_tmp_cleanup_tests {
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-tail-lifecycle";
let object = "object";
for disk in &disks {
disk.make_volume(bucket).await.expect("create volume");
}
let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
let lifecycle = lock
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
.await
.expect("lifecycle read lock");
let metadata = lock
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
.await
.expect("metadata read lock");
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
Uuid::new_v4(),
bucket,
Uuid::new_v4(),
OffsetDateTime::now_utc(),
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
lifecycle,
metadata,
));
let opts = ObjectOptions {
object_lock_config_snapshot: Some(snapshot),
..Default::default()
};
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer_set = Arc::clone(&set);
let put = tokio::spawn(async move {
writer_set
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("rename tail pauses");
put.await.expect("PUT joins").expect("PUT acknowledged");
let mut bucket_delete =
Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
barrier.release();
drop(barrier);
let guard = match early {
Ok(result) => {
drop(result.expect("early bucket lock result"));
panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
}
Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
.await
.expect("lifecycle lock released after tail")
.expect("bucket delete acquires lock"),
};
drop(guard);
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
],
async {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "put-early-ack-tail-read";
let object = "early-ack-tail-read-object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let put_set = Arc::clone(&set_disks);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
put_set
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
.await
.expect("tail disk should pause during rename");
put.await
.expect("PUT task should join")
.expect("PUT should be acknowledged at quorum");
let result = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await;
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
let mut body = Vec::new();
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
.await
.expect("object body should not wait for the tail")
.expect("object body should drain");
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
rename_barrier.release();
drop(rename_barrier);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
@@ -22253,6 +22530,200 @@ mod put_object_tmp_cleanup_tests {
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_does_not_timeout_the_next_put() {
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-early-ack-next-writer";
let object = "object";
for disk in &disks {
disk.make_volume(bucket).await.expect("create volume");
}
{
let map = set.ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
let lifecycle = lifecycle_lock
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
.await
.expect("lifecycle read lock");
let metadata = lifecycle_lock
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
.await
.expect("metadata read lock");
let incarnation = Uuid::new_v4();
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
Uuid::new_v4(),
bucket,
incarnation,
OffsetDateTime::now_utc(),
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
lifecycle,
metadata,
));
let mut opts = ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
object_lock_config_snapshot: Some(snapshot.clone()),
..Default::default()
};
snapshot.add_lock_fences(&mut opts);
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let first_set = Arc::clone(&set);
let first_opts = opts.clone();
let first = tokio::spawn(async move {
first_set
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("first rename paused");
first.await.expect("first PUT joins").expect("first PUT acknowledged");
let second = tokio::time::timeout(
Duration::from_secs(7),
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
)
.await;
barrier.release();
drop(barrier);
let second = second.expect("second PUT finishes").expect("second PUT must not time out");
tokio::time::timeout(Duration::from_secs(30), async {
while rename_tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("both rename tails must drain");
let mut reader = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read newest object");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("read full body");
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
let expected_etag = second.etag.as_ref().expect("second PUT ETag");
for disk in &disks {
let info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("each disk must retain the latest version");
assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_cannot_restore_a_deleted_version() {
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-tail-version-delete";
let object = "object";
for disk in &disks {
disk.make_volume(bucket).await.expect("create volume");
}
{
let map = set.ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
let lifecycle = lock
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
.await
.expect("lifecycle read lock");
let metadata = lock
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
.await
.expect("metadata read lock");
let incarnation = Uuid::new_v4();
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
Uuid::new_v4(),
bucket,
incarnation,
OffsetDateTime::now_utc(),
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
lifecycle,
metadata,
));
let mut opts = ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
object_lock_config_snapshot: Some(snapshot.clone()),
..Default::default()
};
snapshot.add_lock_fences(&mut opts);
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put_opts = opts.clone();
let put = tokio::spawn(async move {
writer
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("rename pauses");
let version = put
.await
.expect("PUT joins")
.expect("PUT succeeds")
.version_id
.expect("version ID")
.to_string();
let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
let delete_set = Arc::clone(&set);
let mut delete_opts = opts;
delete_opts.version_id = Some(version.clone());
let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
delete_barrier.wait_until_paused().await;
delete_barrier.release();
tokio::time::timeout(Duration::from_secs(5), async {
while !delete_barrier.namespace_acquired() {
tokio::task::yield_now().await;
}
})
.await
.expect("version delete acquires namespace while old tail is paused");
barrier.release();
drop(barrier);
delete.await.expect("delete joins").expect("version delete succeeds");
tokio::time::timeout(Duration::from_secs(30), async {
while rename_tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("rename tails drain");
for disk in &disks {
assert!(
matches!(
disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
),
"late tail must not restore the deleted version"
);
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {
@@ -165,6 +165,7 @@ impl SetDisks {
bucket,
object,
opts.no_lock,
false,
DecommissionCapacityAdmission::Mutation,
)
.await?
@@ -275,6 +276,7 @@ impl SetDisks {
bucket,
object,
opts.no_lock,
false,
DecommissionCapacityAdmission::Mutation,
)
.await?
+10 -1
View File
@@ -2294,8 +2294,17 @@ mod tests {
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("fixture rename tail should finish before checking its generation"),
.expect("fixture object lock should be available"),
);
// A quorum-acknowledged rename can release the object lock while its
// physical tail still holds the instance-wide namespace owner.
tokio::time::timeout(Duration::from_secs(30), async {
while ecstore.ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("fixture namespace owners should drain before checking its generation");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(3),
+139 -5
View File
@@ -45,7 +45,8 @@
use crate::error::{Result, StorageError};
use std::collections::HashMap;
use std::future::Future;
use std::sync::{Arc, Mutex};
use std::sync::{Arc, Mutex, Weak};
use std::time::Duration;
use rustfs_lock::NamespaceLockGuard;
use rustfs_lock::distributed_lock::LockLostSignal;
@@ -65,7 +66,7 @@ struct RegisteredGuard {
enum LockLossProbe {
Distributed(Arc<LockLostSignal>),
Local,
Local(Option<rustfs_lock::fast_lock::guard::FastLockGuardProbe>),
#[cfg(test)]
Test(Arc<std::sync::atomic::AtomicBool>),
}
@@ -74,14 +75,19 @@ impl LockLossProbe {
fn from_guard(guard: &NamespaceLockGuard) -> Self {
match guard.lock_lost_signal() {
Some(signal) => Self::Distributed(signal),
None => Self::Local,
None => Self::Local(match guard {
NamespaceLockGuard::Fast(guard) => guard.release_probe(),
NamespaceLockGuard::Standard(_) => None,
}),
}
}
fn is_lost(&self) -> bool {
// Registry callers hold entries before checking a local shard; shard
// operations never acquire the bucket fence registry.
match self {
Self::Distributed(signal) => signal.is_lost(),
Self::Local => false,
Self::Local(probe) => probe.as_ref().is_some_and(|probe| probe.is_lost()),
#[cfg(test)]
Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst),
}
@@ -98,9 +104,74 @@ pub(super) struct FenceRegistration {
#[derive(Default)]
pub(crate) struct BucketFenceRegistry {
entries: Mutex<HashMap<String, FenceEntry>>,
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
}
#[derive(Default)]
pub(super) struct ReservationFenceSlot {
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
}
/// Shared by short bursts of reservations, but never kept alive by the cache.
pub(super) struct ReservationFenceGuard {
publication: NamespaceLockGuard,
lifecycle: BucketIncarnationFenceGuard,
created_at: tokio::time::Instant,
_slot: Arc<ReservationFenceSlot>,
}
impl ReservationFenceGuard {
pub(super) fn new(
publication: NamespaceLockGuard,
lifecycle: BucketIncarnationFenceGuard,
slot: Arc<ReservationFenceSlot>,
) -> Self {
Self {
publication,
lifecycle,
created_at: tokio::time::Instant::now(),
_slot: slot,
}
}
pub(super) fn is_reusable(&self) -> bool {
// Stop admitting new reservations through this shared reader so a
// queued bucket writer can progress after the existing tails drain.
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
}
fn is_held(&self) -> bool {
!self.publication.is_lock_lost()
&& !self.publication.is_released()
&& !self.lifecycle.is_lock_lost()
&& self
.lifecycle
.namespace_lock_guard()
.is_some_and(|guard| !guard.is_released())
}
}
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
fn is_valid(&self) -> bool {
self.is_held()
}
}
impl BucketFenceRegistry {
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
let key = (bucket.to_owned(), incarnation);
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
return slot;
}
if slots.len() >= 1024 {
slots.retain(|_, slot| slot.strong_count() != 0);
}
let slot = Arc::new(ReservationFenceSlot::default());
slots.insert(key, Arc::downgrade(&slot));
slot
}
/// Register a new live guard for `bucket` and return the memoized
/// incarnation id if one is valid for the current coverage window.
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
@@ -234,7 +305,9 @@ impl BucketIncarnationFenceGuard {
}
pub fn is_lock_lost(&self) -> bool {
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
self.inner
.as_ref()
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
}
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
@@ -352,6 +425,67 @@ mod tests {
reg.exit("b", fourth.token, false);
}
#[tokio::test]
async fn disabled_local_lock_does_not_add_a_namespace_fence() {
let manager = Arc::new(rustfs_lock::GlobalLockManager::Disabled(
rustfs_lock::fast_lock::DisabledLockManager::new(),
));
let lock = NamespaceLock::with_local_manager("disabled-fence".into(), manager);
let guard = lock
.get_read_lock(ObjectKey::new("b", "object"), "owner", Duration::from_secs(1))
.await
.expect("disabled lock");
let mut opts = crate::object_api::ObjectOptions::default();
opts.add_namespace_lock_guard(&guard);
assert!(opts.namespace_lock_fence.is_none());
}
#[tokio::test]
async fn forced_local_fence_loss_invalidates_live_incarnation_memo() {
let manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
let global = Arc::new(rustfs_lock::GlobalLockManager::Enabled(manager.clone()));
let lock = NamespaceLock::with_local_manager("forced-fence".into(), global);
let key = ObjectKey::new("b", "lifecycle");
let registry = Arc::new(BucketFenceRegistry::default());
let inner = lock
.get_read_lock(key.clone(), "first", Duration::from_secs(1))
.await
.expect("first lock");
let mut first = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
let mut opts = crate::object_api::ObjectOptions::default();
first.attach_to_object_options(&mut opts);
opts.add_namespace_lock_guard(first.namespace_lock_guard().expect("local guard"));
assert!(
!opts
.bucket_lifecycle_lock_fence
.as_ref()
.expect("lifecycle fence")
.is_lock_lost()
);
assert!(!opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
assert_eq!(manager.force_unlock(&key), 1);
assert!(
opts.bucket_lifecycle_lock_fence
.as_ref()
.expect("attached fence")
.is_lock_lost()
);
assert!(opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
let writer = lock
.get_write_lock(key.clone(), "writer", Duration::from_secs(1))
.await
.expect("writer");
drop(writer);
let inner = lock
.get_read_lock(key, "second", Duration::from_secs(1))
.await
.expect("second lock");
let mut second = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
assert_eq!(second.validate(true, || async { Ok(uuid(8)) }).await.expect("reload"), uuid(8));
assert!(first.validate(true, || async { Ok(uuid(7)) }).await.is_err());
}
#[tokio::test]
async fn fence_guard_forwards_distributed_lock_loss_to_registry() {
let registry = Arc::new(BucketFenceRegistry::default());
+1
View File
@@ -456,6 +456,7 @@ mod rebalance;
pub(crate) use rebalance::PoolSpaceForTest;
pub(crate) mod utils;
pub(crate) use peer::admit_local_disk;
use peer::init_local_peer;
pub use peer::{
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
+1 -1
View File
@@ -1172,7 +1172,7 @@ impl ECStore {
Ok(())
}
async fn check_write_precondition(
pub(crate) async fn check_write_precondition(
&self,
bucket: &str,
object: &str,
File diff suppressed because it is too large Load Diff
+336 -1
View File
@@ -85,6 +85,230 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
}
impl ECStore {
/// Prepare an inert user-object reservation on the currently admitted disk.
/// An unclaimed token expires and does not hold the object's metadata lock.
#[cfg(test)]
pub async fn reserve_local_object_metadata(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None, None)
.await
}
/// Prepare before the caller acquires bucket lifecycle or object locks.
/// The token holds the bucket publication fence through physical completion.
pub async fn reserve_local_object_metadata_at_incarnation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_at_incarnation_with_ttl(disk_ref, bucket, object, incarnation, timeout, None)
.await
}
/// Prepare a short-lived token for peers that may abandon the response.
pub async fn reserve_local_object_metadata_short(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_at_incarnation_with_ttl(
disk_ref,
bucket,
object,
incarnation,
timeout,
Some(std::time::Duration::from_secs(2)),
)
.await
}
async fn reserve_local_object_metadata_at_incarnation_with_ttl(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
ttl: Option<std::time::Duration>,
) -> DiskResult<Uuid> {
if self.ctx.lock_manager().is_disabled()
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|| is_meta_bucketname(bucket)
|| incarnation.is_nil()
|| timeout.is_zero()
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
{
return Err(DiskError::InvalidPath);
}
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
let mut cached = slot.owner.lock().await;
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
Some(owner) => owner,
None => {
// Bucket deletion takes publication before lifecycle and metadata.
let lock = self
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.map_err(|error| DiskError::other(error.to_string()))?;
let publication = lock
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.map_err(|error| DiskError::other(error.to_string()))?;
// Older peers may acquire lifecycle without publication.
let lifecycle = self
.acquire_bucket_incarnation_fence(bucket, incarnation)
.await
.map_err(|error| DiskError::other(error.to_string()))?;
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
if !owner.is_reusable() {
return Err(DiskError::DiskOngoingReq);
}
*cached = Arc::downgrade(&owner);
owner
}
};
drop(cached);
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner), ttl)
.await
})
.await
.map_err(|_| DiskError::Timeout)?
}
async fn reserve_local_object_metadata_with_owner(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
timeout: std::time::Duration,
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
ttl: Option<std::time::Duration>,
) -> DiskResult<Uuid> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
return Err(DiskError::InvalidPath);
}
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
match ttl {
Some(ttl) => {
wrapper
.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
.await
}
None => wrapper.prepare_object_metadata_with_owner(bucket, object, owner).await,
}
})
.await
.map_err(|_| DiskError::Timeout)?
}
/// Refresh an inert token on its original disk and object.
pub async fn renew_local_object_metadata_reservation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
token: Uuid,
) -> DiskResult<()> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
return Err(DiskError::InvalidPath);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
wrapper.renew_object_metadata(bucket, object, token).await
}
/// Atomically enqueue a prepared token on its original disk and object.
pub async fn activate_local_object_metadata_reservation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
token: Uuid,
) -> DiskResult<()> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
return Err(DiskError::InvalidPath);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
wrapper.activate_object_metadata(bucket, object, token).await
}
/// Cancel an unclaimed token on the currently admitted disk.
/// A token already claimed by a physical mutation cannot be cancelled.
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
Ok(wrapper.cancel_object_metadata_reservation(token))
}
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
/// The token is bound to the admitted disk instance and destination. A claimed
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
/// The caller must retain the bucket lifecycle fence until the physical tail
/// finishes; the reservation does not identify a bucket incarnation.
pub async fn rename_local_data_reserved(
&self,
disk_ref: &str,
source: (&str, &str),
fi: &FileInfo,
destination: (&str, &str),
token: Uuid,
) -> DiskResult<RenameDataResp> {
if token.is_nil()
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|| is_meta_bucketname(destination.0)
|| rustfs_utils::http::metadata_compat::contains_key_str(
&fi.metadata,
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
)
{
return Err(DiskError::InvalidPath);
}
rename_local_data_with_ctx(
&self.ctx,
disk_ref,
source,
fi,
destination,
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
)
.await
}
/// Resolve a quota lease against this request's captured storage instance.
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
@@ -270,6 +494,7 @@ impl ECStore {
scanner_publication_lease_token: scanner_token,
external_guard,
namespace_owner: None,
..Default::default()
},
)
.await
@@ -351,7 +576,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
Ok(())
}
async fn admit_local_disk(
pub(crate) async fn admit_local_disk(
ctx: &Arc<InstanceContext>,
disk: &DiskStore,
disk_id: Option<Uuid>,
@@ -783,6 +1008,116 @@ mod tests {
.expect("seeded metadata bytes")
}
#[tokio::test]
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
let ctx = Arc::new(InstanceContext::new());
let dir = tempfile::tempdir().expect("disk root");
let id = Uuid::new_v4();
let disk = target_disk(&ctx, dir.path(), id).await;
let disk_ref = disk.endpoint().to_string();
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let version = Uuid::new_v4();
let fi = target_file_info("object", version, b"reserved");
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
.await
.expect("user bucket");
assert!(matches!(
store
.reserve_local_object_metadata(
&disk_ref,
"target-bucket/../.rustfs.sys",
"object",
std::time::Duration::from_secs(5),
)
.await,
Err(DiskError::InvalidPath)
));
assert!(matches!(
store
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
.await,
Err(DiskError::InvalidPath)
));
let token = store
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
.await
.expect("reserve current disk");
assert!(matches!(
store
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
.await,
Err(DiskError::InvalidPath)
));
store
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
.await
.expect("activate token");
store
.rename_local_data_reserved(
&disk_ref,
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
&fi,
("target-bucket", "object"),
token,
)
.await
.expect("reserved publication");
assert!(
!store
.cancel_local_object_metadata_reservation(&disk_ref, token)
.await
.expect("consumed token")
);
let read = disk
.read_version(
"target-bucket",
"target-bucket",
"object",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read reserved publication");
assert_eq!(read.data, fi.data);
let stale = store
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
.await
.expect("reserve before disk replacement");
let replacement = target_disk(&ctx, dir.path(), id).await;
assert!(matches!(
store
.rename_local_data_reserved(
&disk_ref,
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
&fi,
("target-bucket", "object"),
stale,
)
.await,
Err(DiskError::DiskOngoingReq)
));
let after = replacement
.read_version(
"target-bucket",
"target-bucket",
"object",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("replacement retains publication");
assert_eq!(after.data, fi.data);
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
}
#[tokio::test]
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
for warm in [false, true] {
+18
View File
@@ -604,6 +604,24 @@ impl ECStore {
Ok(idx)
}
/// A speculative hint only: callers must repeat the locked selection before committing.
pub(super) async fn get_pool_idx_for_reservation_hint(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
let opts = ObjectOptions {
no_lock: true,
metadata_cache_safe: true,
skip_decommissioned: true,
skip_rebalancing: true,
..Default::default()
};
match self.get_pool_idx_existing_with_opts(bucket, object, &opts).await {
Ok(idx) => Ok(idx),
Err(err) if is_err_object_not_found(&err) => {
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
}
Err(err) => Err(err),
}
}
pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await {
Ok(res) => res,
-68
View File
@@ -3775,74 +3775,6 @@ mod tests {
assert!(corrupt_crc_entry.is_latest_delete_marker());
}
#[test]
fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
.expect("load realistic xl.meta");
let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
let mut free_meta_sys = HashMap::new();
rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
let free = FileMetaShallowVersion::try_from(FileMetaVersion {
version_type: VersionType::Delete,
delete_marker: Some(MetaDeleteMarker {
version_id: Some(Uuid::from_u128(42)),
mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
meta_sys: free_meta_sys,
}),
write_version: 1,
..Default::default()
})
.expect("encode cleanup record");
fm.versions.insert(0, free);
let entry = MetaCacheEntry {
name: "object".to_string(),
metadata: fm.marshal_msg().expect("encode metadata"),
cached: None,
reusable: false,
};
let scanned = fm
.get_file_info_versions("bucket", "object", false)
.expect("scanner version decode");
assert_eq!(scanned.free_versions.len(), 1);
assert_eq!(scanned.versions.len(), 1);
assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
assert_eq!(listed.version_id, object.version_id);
assert_eq!(listed.version_id, scanned.versions[0].version_id);
assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
assert_eq!(listed.size, scanned.versions[0].size);
assert!(!listed.deleted);
let marker_id = Uuid::from_u128(43);
fm.add_version(FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(marker_id),
deleted: true,
mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
..Default::default()
})
.expect("add current delete marker");
let mut marker_entry = MetaCacheEntry {
name: "object".to_string(),
metadata: fm.marshal_msg().expect("encode marker metadata"),
cached: None,
reusable: false,
};
let scanned = fm
.get_file_info_versions("bucket", "object", false)
.expect("scan marker stack");
assert_eq!(scanned.versions.len(), 2);
assert_eq!(scanned.versions[0].version_id, Some(marker_id));
assert!(scanned.versions[0].deleted);
assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
assert!(marker_entry.is_latest_delete_marker());
let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
assert_eq!(listed.version_id, scanned.versions[0].version_id);
assert!(listed.deleted);
}
#[test]
fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
+68
View File
@@ -27,6 +27,7 @@ use futures::{StreamExt, stream::FuturesUnordered};
use metrics::{counter, gauge};
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
use rustfs_madmin::heal_commands::HealResultItem;
use std::collections::HashMap;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
@@ -1550,6 +1551,7 @@ impl ErasureSetHealer {
let checkpoint = checkpoint_manager.get_checkpoint().await;
let page_resume_index = *current_object_index;
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
let mut object_locks: HashMap<String, Arc<tokio::sync::Mutex<()>>> = HashMap::new();
let mut page_tasks = FuturesUnordered::new();
let mut completed_in_page = 0usize;
@@ -1721,6 +1723,7 @@ impl ErasureSetHealer {
let set_label = set_disk_id.to_string();
let heal_opts = self.heal_opts;
let semaphore = semaphore.clone();
let object_lock = object_locks.entry(object_name.clone()).or_default().clone();
let target_endpoints = self.target_endpoints.clone();
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
@@ -1735,6 +1738,13 @@ impl ErasureSetHealer {
.as_ref()
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
let work = async move {
// Versions share one namespace lock. Queue them before taking a
// page permit so a waiting version cannot block other objects.
let _object_guard = tokio::select! {
biased;
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
guard = object_lock.lock() => guard,
};
let mut retries = 0;
let mut admin_identity = admin_identity;
let mut admin_task_id = admin_task_id;
@@ -4047,6 +4057,64 @@ mod resume_loop_tests {
assert_eq!(unique.len(), 50, "no version healed twice");
}
#[tokio::test]
async fn page_serializes_versions_without_blocking_other_objects() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("true")),
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
],
async {
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer = env
.healer
.with_replacement_targets(vec!["replacement-a".into()], Some(env.task_id.clone()));
let gate = Arc::new(tokio::sync::Semaphore::new(0));
*env.storage.heal_gate.lock().unwrap() = Some(gate.clone());
for (name, version) in [("a", "v1"), ("a", "v2"), ("b", "v1")] {
env.storage
.set_result(name, Some(version), replacement_target_ok_result("replacement-a", name));
}
env.storage.set_page(
None,
Page {
items: vec![
item("a", Some("v1"), false),
item("a", Some("v2"), false),
item("b", Some("v1"), false),
],
next: None,
truncated: false,
},
);
let observe = async {
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
let notified = env.storage.heal_started.notified();
if env.storage.calls().len() >= 2 {
break;
}
notified.await;
}
})
.await
.is_ok();
let calls = env.storage.calls();
gate.add_permits(3);
assert!(started, "different objects should run concurrently");
assert_eq!(calls.len(), 2);
assert!(calls.iter().any(|(name, _)| name == "a"));
assert!(calls.iter().any(|(name, _)| name == "b"));
};
let ((processed, successful, failed, skipped, result), ()) = tokio::join!(run(&env), observe);
result.expect("all versions heal");
assert_eq!((processed, successful, failed, skipped), (3, 3, 0, 0));
},
)
.await;
}
#[tokio::test]
async fn auto_replacement_runs_two_object_repairs_within_the_configured_limit() {
temp_env::async_with_vars(
+312 -121
View File
@@ -14,13 +14,26 @@
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
use super::*;
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
use crate::heal::storage::HealListItem;
use crate::heal::storage::{HealListItem, HealStorageObjectResult};
use crate::heal::utils::format_set_disk_id;
use futures::{StreamExt, stream::FuturesUnordered};
use std::collections::HashSet;
const MAX_DEFERRED_OBJECTS: usize = 256;
const MAX_DEFERRED_BYTES: usize = 256 * 1024;
const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
const MAX_INFLIGHT_REPAIRS: usize = 8;
#[cfg(test)]
pub(super) fn deferred_payload_budget_for_test() -> usize {
MAX_DEFERRED_BYTES - MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>()
}
// A recovered pool can become writable just after the ordinary retry budget.
// Keep this extra attempt inside the existing deferred age and backoff caps.
const MAX_SLOWDOWN_OBJECT_RETRIES: u32 = MAX_BUCKET_OBJECT_HEAL_RETRIES + 1;
type HealListingPage = (Vec<HealListItem>, Option<String>, bool);
struct DeferredObject {
name: String,
@@ -57,7 +70,11 @@ impl DeferredObject {
let now = tokio::time::Instant::now();
let first = *self.first_failure.get_or_insert(now);
self.attempt += 1;
self.due = (now + delay).min(first + MAX_DEFERRED_AGE);
let remaining = (first + MAX_DEFERRED_AGE).saturating_duration_since(now);
// Scheduling exactly at the deadline guarantees the retry will be
// rejected as expired without making the storage attempt.
let delay = if delay >= remaining { remaining / 2 } else { delay };
self.due = now + delay;
}
}
@@ -89,16 +106,23 @@ impl DeferredWindow {
Ok(())
}
fn pop_due(&mut self) -> Option<DeferredObject> {
fn pop_due(&mut self, in_flight: &HashSet<String>) -> Option<DeferredObject> {
let now = tokio::time::Instant::now();
let index = self.objects.iter().position(|item| item.due <= now)?;
let index = self
.objects
.iter()
.position(|item| item.due <= now && !in_flight.contains(&item.name))?;
let item = self.objects.remove(index)?;
self.bytes -= item.payload_bytes();
Some(item)
}
fn next_due(&self) -> Option<tokio::time::Instant> {
self.objects.iter().map(|item| item.due).min()
fn next_due(&self, in_flight: &HashSet<String>) -> Option<tokio::time::Instant> {
self.objects
.iter()
.filter(|item| !in_flight.contains(&item.name))
.map(|item| item.due)
.min()
}
fn can_advance(&self, page: u64) -> bool {
@@ -132,6 +156,32 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
}
impl HealTask {
async fn heal_bucket_storage_object(
&self,
bucket: &str,
item: DeferredObject,
incarnation: Option<Uuid>,
opts: &HealOpts,
) -> (DeferredObject, Result<HealStorageObjectResult>) {
let result = self
.await_with_control(async {
match incarnation {
Some(expected) => {
self.storage
.heal_object_at_incarnation(bucket, &item.name, item.version_id.as_deref(), expected, opts)
.await
}
None => {
self.storage
.heal_object_with_receipt(bucket, &item.name, item.version_id.as_deref(), opts)
.await
}
}
})
.await;
(item, result)
}
async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result<bool> {
if let Some(result) = self
.await_with_control(
@@ -649,56 +699,168 @@ impl HealTask {
};
let mut continuation_token: Option<String> = None;
let mut deferred = DeferredWindow::default();
let mut inline_retry: Option<DeferredObject> = None;
// Only already-admitted operations can overflow the deferred byte
// budget. Stop listing and page advancement while these drain.
let mut inline_retry = VecDeque::<DeferredObject>::new();
let mut page_number = 0_u64;
let mut aborted_progress_unknown = false;
let mut pending = Vec::<HealListItem>::new().into_iter();
let mut listing_finished = false;
let mut listing_attempt = 0;
let mut listing_due = tokio::time::Instant::now();
let mut listing_inflight: Option<futures::future::BoxFuture<'static, Result<HealListingPage>>> = None;
let mut in_flight = HashSet::new();
let repairs_aborted = Arc::new(AtomicBool::new(false));
let mut repairs: FuturesUnordered<
futures::future::BoxFuture<'_, (DeferredObject, Result<HealStorageObjectResult>, bool)>,
> = FuturesUnordered::new();
let scope_result: Result<()> = async {
loop {
self.check_control_flags().await?;
if listing_finished && pending.as_slice().is_empty() && deferred.objects.is_empty() && inline_retry.is_none()
if listing_finished
&& pending.as_slice().is_empty()
&& deferred.objects.is_empty()
&& inline_retry.is_empty()
&& repairs.is_empty()
{
break;
}
self.pace_mainline().await?;
// Listing and object retries share this safe boundary. A
// failed listing never hides an already-due object retry.
let item = deferred.pop_due().or_else(|| {
if inline_retry
.as_ref()
.is_some_and(|item| item.due <= tokio::time::Instant::now())
{
inline_retry.take()
} else if inline_retry.is_none() {
let item = if repairs.len() < MAX_INFLIGHT_REPAIRS {
let due = inline_retry
.iter()
.position(|item| item.due <= tokio::time::Instant::now() && !in_flight.contains(&item.name));
due.and_then(|index| inline_retry.remove(index))
.or_else(|| deferred.pop_due(&in_flight))
} else {
None
}
.or_else(|| {
if inline_retry.is_empty() && repairs.is_empty() {
pending.next().map(|item| DeferredObject::new(item, page_number))
} else {
None
}
});
let Some(mut item) = item else {
let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number);
let (mut item, storage_result, age_exhausted) = if let Some(item) = item {
if item.expired() {
(item, Err(Error::other("heal object retry age exhausted")), true)
} else {
in_flight.insert(item.name.clone());
let opts = &heal_opts;
let aborted = Arc::clone(&repairs_aborted);
repairs.push(Box::pin(async move {
if aborted.load(Ordering::Acquire) {
return (item, Err(Error::TaskCancelled), false);
}
if let Err(error) = self.pace_mainline().await {
return (item, Err(error), false);
}
if aborted.load(Ordering::Acquire) {
return (item, Err(Error::TaskCancelled), false);
}
let (item, result) = self
.heal_bucket_storage_object(bucket, item, traversal_incarnation_id, opts)
.await;
(item, result, true)
}));
continue;
}
} else if !repairs.is_empty() {
let next_retry = if repairs.len() < MAX_INFLIGHT_REPAIRS {
deferred
.next_due(&in_flight)
.into_iter()
.chain(
inline_retry
.iter()
.filter(|item| !in_flight.contains(&item.name))
.map(|item| item.due),
)
.min()
} else {
None
};
let completed = if let Some(due) = next_retry {
tokio::select! {
biased;
_ = tokio::time::sleep_until(due) => None,
completed = repairs.next() => completed,
}
} else {
repairs.next().await
};
let Some((item, result, attempted)) = completed else {
continue;
};
in_flight.remove(&item.name);
if !attempted && item.attempt == 0 {
return Err(result.expect_err("an unstarted repair must have a control error"));
}
(item, result, false)
} else {
self.pace_mainline().await?;
let can_list = !listing_finished && inline_retry.is_empty() && deferred.can_advance(page_number);
if can_list && listing_due <= tokio::time::Instant::now() {
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
set_disk_id,
bucket,
prefix,
continuation_token.as_deref(),
false,
))
.await
} else {
self.await_with_control(self.storage.list_objects_for_heal_page(
bucket,
prefix,
continuation_token.as_deref(),
false,
))
.await
if listing_inflight.is_none() {
let storage = Arc::clone(&self.storage);
let listing_set = set_disk_id.clone();
let listing_bucket = bucket.to_owned();
let listing_prefix = prefix.to_owned();
let cursor = continuation_token.clone();
listing_inflight = Some(Box::pin(async move {
if let Some(set_id) = listing_set.as_deref() {
storage
.list_versions_for_heal_page_disk_walk(
set_id,
&listing_bucket,
&listing_prefix,
cursor.as_deref(),
false,
)
.await
} else {
storage
.list_objects_for_heal_page(
&listing_bucket,
&listing_prefix,
cursor.as_deref(),
false,
)
.await
}
}));
}
let next_retry = deferred
.next_due(&in_flight)
.into_iter()
.chain(inline_retry.iter().map(|item| item.due))
.min();
let Some(listing) = listing_inflight.as_mut() else {
return Err(Error::other("heal listing future is unavailable"));
};
// Keep the same listing future alive while an earlier retry runs.
// Restarting a disk walk here would duplicate work and may change its snapshot.
let result = self
.await_with_control(async {
if let Some(due) = next_retry {
tokio::select! {
biased;
_ = tokio::time::sleep_until(due) => Ok(None),
result = listing => Ok(Some(result)),
}
} else {
Ok(Some(listing.await))
}
})
.await;
let page = match result {
Ok(Some(page)) => page,
Ok(None) => continue,
Err(error) => Err(error),
};
listing_inflight = None;
match page {
Ok((objects, next_token, is_truncated)) => {
page_number = page_number.saturating_add(1);
@@ -727,9 +889,9 @@ impl HealTask {
continue;
} else {
let due = deferred
.next_due()
.next_due(&in_flight)
.into_iter()
.chain(inline_retry.as_ref().map(|item| item.due))
.chain(inline_retry.iter().map(|item| item.due))
.chain(can_list.then_some(listing_due))
.min();
if let Some(due) = due {
@@ -761,89 +923,74 @@ impl HealTask {
}
let mut terminal_outcome = true;
let age_exhausted = item.expired();
let error = if age_exhausted {
Some(Error::other("heal object retry age exhausted"))
} else {
match self
.await_with_control(async {
match traversal_incarnation_id {
Some(expected) => {
self.storage
.heal_object_at_incarnation(
bucket,
object,
item.version_id.as_deref(),
expected,
&heal_opts,
)
.await
}
None => {
self.storage
.heal_object_with_receipt(bucket, object, item.version_id.as_deref(), &heal_opts)
.await
}
}
})
.await
{
Ok(storage_result) if storage_result.error.is_none() => {
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
Some(error) => Some(error),
None => {
#[cfg(test)]
super::pause_storage_receipt(&self.id).await;
recorded_authoritative_outcome =
self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
Some(Error::TaskCancelled)
} else {
telemetry_unknown |= !increment_counter(&mut healed);
telemetry_unknown |= !add_bytes(
&mut bytes,
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
);
if !recorded_authoritative_outcome
&& !heal_opts.dry_run
&& !storage_result.item.detail.is_empty()
{
detail = Some(storage_result.item.detail.clone());
}
self.record_result_item(storage_result.item).await;
None
let object = item.name.as_str();
let error = match storage_result {
Ok(storage_result) if storage_result.error.is_none() => {
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
Some(error) => Some(error),
None => {
#[cfg(test)]
super::pause_storage_receipt(&self.id).await;
recorded_authoritative_outcome =
self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
Some(Error::TaskCancelled)
} else {
telemetry_unknown |= !increment_counter(&mut healed);
telemetry_unknown |= !add_bytes(
&mut bytes,
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
);
if !recorded_authoritative_outcome
&& !heal_opts.dry_run
&& !storage_result.item.detail.is_empty()
{
detail = Some(storage_result.item.detail.clone());
}
self.record_result_item(storage_result.item).await;
None
}
}
}
Ok(storage_result)
if storage_result
.error
.as_ref()
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
{
telemetry_unknown |= !increment_counter(&mut healed);
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_TASK,
task_id = %self.id,
bucket,
object,
result = "object_dir_not_found_skipped",
"Heal bucket object-dir candidate skipped after not-found result"
);
None
}
Ok(storage_result) => storage_result.error,
Err(err) => Some(err),
}
Ok(storage_result)
if storage_result
.error
.as_ref()
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
{
telemetry_unknown |= !increment_counter(&mut healed);
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_TASK,
task_id = %self.id,
bucket,
object,
result = "object_dir_not_found_skipped",
"Heal bucket object-dir candidate skipped after not-found result"
);
None
}
Ok(storage_result) => storage_result.error,
Err(err) => Some(err),
};
if let Some(err) = error {
match err {
Error::StaleBucketIncarnation { .. } => return Err(err),
Error::StaleBucketIncarnation { .. } => {
self.outcome.write().await.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity,
disposition: HealObjectDisposition::Unknown,
detail: None,
});
aborted_progress_unknown |= !increment_counter(&mut scanned);
aborted_progress_unknown |= !increment_counter(&mut skipped);
return Err(err);
}
Error::TaskCancelled | Error::TaskTimeout => {
let disposition = if matches!(err, Error::TaskCancelled) {
HealObjectDisposition::Cancelled
@@ -912,7 +1059,12 @@ impl HealTask {
);
} else if !age_exhausted
&& err.is_recoverable_object_heal()
&& retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES
&& retry_attempt
< if matches!(&err, Error::Storage(crate::heal::StorageError::SlowDown)) {
MAX_SLOWDOWN_OBJECT_RETRIES
} else {
MAX_BUCKET_OBJECT_HEAL_RETRIES
}
{
terminal_outcome = false;
debug!(
@@ -930,7 +1082,7 @@ impl HealTask {
);
item.defer(Self::bucket_object_retry_delay(&self.id, retry_attempt + 1));
if let Err(item) = deferred.push(item) {
inline_retry = Some(item);
inline_retry.push_back(item);
}
} else {
disposition = HealObjectDisposition::Failed(if age_exhausted || err.is_recoverable_object_heal() {
@@ -947,17 +1099,24 @@ impl HealTask {
first_failed_object.get_or_insert_with(|| object.to_string());
first_error.get_or_insert_with(|| err.to_string());
if take_failure_log_sample(&mut failure_samples_logged) {
let retry_age_ms = item
.first_failure
.map(|first| u64::try_from(first.elapsed().as_millis()).unwrap_or(u64::MAX));
let retry_due_lag_ms =
age_exhausted.then(|| u64::try_from(item.due.elapsed().as_millis()).unwrap_or(u64::MAX));
warn!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_TASK,
result = "object_failed",
task_id = %self.id,
bucket,
object,
retry_attempt,
retry_age_ms = ?retry_age_ms,
retry_due_lag_ms = ?retry_due_lag_ms,
error = %err,
result = "object_failed",
"Heal bucket object repair failed"
);
}
@@ -998,6 +1157,7 @@ impl HealTask {
}
.await;
if let Err(error) = scope_result {
repairs_aborted.store(true, Ordering::Release);
let disposition = match error {
Error::TaskCancelled => HealObjectDisposition::Cancelled,
Error::TaskTimeout => HealObjectDisposition::Deferred {
@@ -1006,20 +1166,51 @@ impl HealTask {
},
_ => HealObjectDisposition::Unknown,
};
// Keep issued replacement I/O alive until it reaches its storage
// boundary, even when another operation aborts this traversal.
while let Some((item, result, attempted)) = repairs.next().await {
if !attempted && item.attempt == 0 {
continue;
}
let mut identity =
self.outcome_identity(bucket, &item.name, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
identity.bucket_incarnation_id = bucket_incarnation_id;
let mut recorded = false;
if let Ok(storage_result) = result
&& storage_result.error.is_none()
&& unavailable_recreate_error(&storage_result.item, &heal_opts).is_none()
{
recorded = self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
if recorded {
aborted_progress_unknown |= !increment_counter(&mut healed);
aborted_progress_unknown |=
!add_bytes(&mut bytes, u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX));
self.record_result_item(storage_result.item).await;
}
}
if !recorded {
self.outcome.write().await.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity,
disposition: disposition.clone(),
detail: None,
});
aborted_progress_unknown |= !increment_counter(&mut skipped);
}
aborted_progress_unknown |= !increment_counter(&mut scanned);
}
// Only attempted identities have terminal outcomes. Unstarted
// page tails remain unprocessed under the task's partial coverage.
// No detached sleepers survive abort.
for item in deferred.objects.into_iter().chain(inline_retry) {
let mut identity =
self.outcome_identity(bucket, &item.name, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
identity.bucket_incarnation_id = bucket_incarnation_id;
self.outcome.write().await.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: self.outcome_identity(
bucket,
&item.name,
item.version_id.as_deref(),
heal_opts.pool,
heal_opts.set,
),
identity,
disposition: disposition.clone(),
detail: None,
});
+18 -2
View File
@@ -1658,7 +1658,9 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
struct MockStorage {
retry_test_pages: Option<Vec<Vec<HealListItem>>>,
retry_test_delays: HashMap<String, Duration>,
retry_test_attempt_delays: Mutex<HashMap<String, VecDeque<Duration>>>,
retry_test_listing_delays: Mutex<VecDeque<Duration>>,
retry_test_disk_walk_pages: bool,
retry_test_events: Mutex<Vec<String>>,
listed: Mutex<bool>,
list_each_bucket: bool,
@@ -1678,6 +1680,7 @@ struct MockStorage {
object_heal_detail: Option<String>,
object_heal_item: Option<HealResultItem>,
bucket_incarnation_id: Mutex<Option<Uuid>>,
bucket_incarnation_reads: std::sync::atomic::AtomicUsize,
bucket_incarnation_after_object_heal: Mutex<Option<Uuid>>,
bucket_incarnation_after_listing: Mutex<Option<Uuid>>,
bucket_incarnation_unavailable: Mutex<bool>,
@@ -2427,6 +2430,7 @@ impl HealStorageAPI for MockStorage {
}
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
self.bucket_incarnation_reads.fetch_add(1, Ordering::Relaxed);
if *self.bucket_incarnation_unavailable.lock().unwrap() {
return Err(Error::Other("bucket incarnation unavailable".to_string()));
}
@@ -2459,8 +2463,15 @@ impl HealStorageAPI for MockStorage {
) -> Result<(HealResultItem, Option<Error>)> {
self.heal_object_calls.lock().unwrap().push(object.to_string());
self.retry_test_events.lock().expect("events").push(format!("heal:{object}"));
if let Some(delay) = self.retry_test_delays.get(object) {
tokio::time::sleep(*delay).await;
let delay = self
.retry_test_attempt_delays
.lock()
.expect("attempt delays")
.get_mut(object)
.and_then(VecDeque::pop_front)
.or_else(|| self.retry_test_delays.get(object).copied());
if let Some(delay) = delay {
tokio::time::sleep(delay).await;
}
self.heal_object_version_ids
.lock()
@@ -2913,6 +2924,11 @@ impl HealStorageAPI for MockStorage {
_include_lifecycle_object_info: bool,
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
self.disk_walk_calls.lock().unwrap().push(set_disk_id.to_string());
if self.retry_test_disk_walk_pages {
return self
.list_objects_for_heal_page(_bucket, _prefix, continuation_token, _include_lifecycle_object_info)
.await;
}
if continuation_token.is_some() {
return Ok((Vec::new(), None, false));
}
+499 -29
View File
@@ -52,7 +52,92 @@ fn fail_once(storage: &MockStorage, name: &str) {
}
#[tokio::test(start_paused = true)]
async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
async fn slow_listing_does_not_starve_due_object_retry() {
let storage = Arc::new(MockStorage {
recoverable_second_page_failures: Mutex::new(Some(1)),
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
..Default::default()
});
fail_once(&storage, "object-a");
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
["list:first", "heal:object-a", "list:second", "heal:object-a"],
"the object retry must run while the slow listing remains in flight"
);
assert_eq!(task.get_outcome().await.counters.failed, 0);
execution.await.expect("the resumed listing and retried object complete");
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
[
"list:first",
"heal:object-a",
"list:second",
"heal:object-a",
"list:second",
"heal:object-b"
]
);
}
#[tokio::test(start_paused = true)]
async fn slow_set_disk_walk_does_not_starve_due_object_retry() {
let storage = Arc::new(MockStorage {
erasure_set_scopes: Mutex::new(vec![(0, 0)]),
retry_test_disk_walk_pages: true,
retry_test_pages: Some(vec![vec![heal_item("a")], vec![heal_item("b")]]),
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
..Default::default()
});
fail_once(&storage, "a");
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
["list:first", "heal:a", "list:1", "heal:a"]
);
assert_eq!(storage.disk_walk_calls.lock().expect("walks").len(), 2);
execution.await.expect("the same disk walk resumes and both objects heal");
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "a", "b"]);
assert_eq!(storage.disk_walk_calls.lock().expect("walks").len(), 2);
assert_eq!(task.get_outcome().await.counters.failed, 0);
}
#[tokio::test(start_paused = true)]
async fn cancelling_an_inflight_listing_keeps_deferred_outcome_cancelled() {
let incarnation = Uuid::new_v4();
let storage = Arc::new(MockStorage {
recoverable_second_page_failures: Mutex::new(Some(1)),
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
bucket_incarnation_id: Mutex::new(Some(incarnation)),
..Default::default()
});
fail_once(&storage, "object-a");
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
["list:first", "heal:object-a", "list:second"]
);
task.cancel().await.expect("cancel");
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
tokio::time::advance(Duration::from_secs(60)).await;
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"]);
let outcome = task.get_outcome().await;
assert_eq!(outcome.objects.len(), 1);
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
}
#[tokio::test(start_paused = true)]
async fn slow_listing_services_multiple_retries_without_restarting_walk() {
let storage = Arc::new(MockStorage {
recoverable_second_page_failures: Mutex::new(Some(1)),
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(29)])),
@@ -68,22 +153,17 @@ async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(
tokio::time::timeout(Duration::from_millis(30_500), &mut execution)
.await
.is_err()
);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
["list:first", "heal:object-a", "list:second", "heal:object-a"]
["list:first", "heal:object-a", "list:second", "heal:object-a", "heal:object-a"]
);
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.processed, 1);
assert_eq!(
outcome.objects[0].disposition,
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(1, 0, 2)
);
execution.await.expect_err("age exhausted object must remain a batch failure");
execution.await.expect("object and resumed listing complete");
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
[
@@ -91,22 +171,25 @@ async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
"heal:object-a",
"list:second",
"heal:object-a",
"heal:object-a",
"list:second",
"heal:object-b"
]
);
let outcome = task.get_outcome().await;
assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 3));
}
#[tokio::test(start_paused = true)]
async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() {
async fn slow_listing_preserves_retry_limit_and_failure() {
let storage = Arc::new(MockStorage {
recoverable_second_page_failures: Mutex::new(Some(1)),
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
..Default::default()
});
fail_once(&storage, "object-a");
storage
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert("object-a".to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
@@ -115,15 +198,20 @@ async fn listing_return_after_age_expires_does_not_start_another_heal_attempt()
.await
.is_err()
);
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"; 5]);
let outcome = task.get_outcome().await;
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
["list:first", "heal:object-a", "list:second"]
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(1, 1, 6) // Five object failures and one listing failure.
);
assert_eq!(task.get_outcome().await.counters.failed, 1);
execution.await.expect_err("age exhaustion remains a failure");
assert_eq!(
storage.retry_test_events.lock().expect("events").as_slice(),
["list:first", "heal:object-a", "list:second", "list:second", "heal:object-b"]
outcome.objects[0].disposition,
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
);
execution.await.expect_err("exhausted object must remain a batch failure");
assert_eq!(
storage.heal_object_calls.lock().expect("calls").last().map(String::as_str),
Some("object-b")
);
}
@@ -220,6 +308,29 @@ async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unproc
}
}
#[tokio::test(start_paused = true)]
async fn slowdown_can_recover_within_existing_deferred_age_budget() {
let storage = Arc::new(pages_storage(&[&["a"]]));
storage
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert("a".to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
let task = bucket_task(storage.clone());
let started = tokio::time::Instant::now();
task.execute()
.await
.expect("transient slowdown recovers within the deferred age budget");
assert!(started.elapsed() < Duration::from_secs(30));
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 5);
let outcome = task.get_outcome().await;
assert_eq!(
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(1, 0, 4)
);
assert_eq!(outcome.objects.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
@@ -228,7 +339,7 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
.insert(name.to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
}
let task = bucket_task(storage.clone());
let execution = task.execute();
@@ -239,9 +350,9 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
let outcome = task.get_outcome().await;
assert_eq!(
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(4, 4, 16)
(4, 4, 20)
);
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 20);
for name in ["a", "b", "c", "d"] {
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
}
@@ -294,14 +405,360 @@ async fn due_retry_runs_before_next_object_in_a_slow_healthy_page() {
}
#[tokio::test(start_paused = true)]
async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
let mut storage = pages_storage(&[&["a"], &["b"]]);
async fn slow_unrelated_object_does_not_starve_due_retry() {
let mut storage = pages_storage(&[&["a", "b"]]);
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
fail_once(&storage, "a");
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
task.execute().await.expect_err("aged pending responsibility is not success");
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(
storage.heal_object_calls.lock().expect("calls").as_slice(),
["a", "b", "a"],
"a due retry must run while an unrelated object is still healing"
);
execution.await.expect("both objects complete");
let outcome = task.get_outcome().await;
assert_eq!((outcome.counters.processed, outcome.counters.failed), (2, 0));
}
#[tokio::test(start_paused = true)]
async fn cancellation_drains_slow_replacement_object_after_a_concurrent_retry() {
let mut storage = pages_storage(&[&["a", "b"]]);
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
storage
.retry_test_attempt_delays
.lock()
.expect("attempt delays")
.insert("a".to_string(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)]));
fail_once(&storage, "a");
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
let owner = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
*task.replacement_execution.write().await = Some(owner);
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a"]);
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 1);
task.cancel_token.cancel();
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
assert!(task.replacement_execution.read().await.is_none());
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a"]);
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 3);
}
#[tokio::test(start_paused = true)]
async fn stale_bucket_retry_drains_another_inflight_replacement() {
let original = Uuid::new_v4();
let mut storage = pages_storage(&[&["a", "b"]]);
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original);
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
fail_once(&storage, "a");
let storage = Arc::new(storage);
let mut task = bucket_task(storage.clone());
task.bucket_incarnation_id = Some(original);
let owner = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
*task.replacement_execution.write().await = Some(owner);
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4());
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
let result = execution.await;
assert!(
matches!(result, Err(Error::StaleBucketIncarnation { .. })),
"unexpected result: {result:?}"
);
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 2);
assert!(task.replacement_execution.read().await.is_none());
let outcome = task.get_outcome().await;
let stale = outcome
.objects
.iter()
.find(|item| item.identity.object == "a")
.expect("stale retry outcome");
assert_eq!(stale.identity.bucket_incarnation_id, Some(original));
assert_eq!(stale.disposition, HealObjectDisposition::Unknown);
let other = outcome
.objects
.iter()
.find(|item| item.identity.object == "b")
.expect("b outcome");
assert_eq!(other.identity.bucket_incarnation_id, Some(original));
assert_eq!(other.disposition, HealObjectDisposition::Unknown);
}
#[tokio::test(start_paused = true)]
async fn stale_retry_preserves_verified_receipt_from_another_inflight_replacement() {
let original = Uuid::new_v4();
let mut storage = pages_storage(&[&["a", "b"]]);
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original);
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
storage.heal_object_receipts.lock().expect("receipts").insert(
"b".to_string(),
VecDeque::from([object_receipt("b", None, HealObjectDisposition::Repaired, original)]),
);
fail_once(&storage, "a");
let storage = Arc::new(storage);
let mut task = bucket_task(storage.clone());
task.bucket_incarnation_id = Some(original);
*task.replacement_execution.write().await =
Some(crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new()));
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4());
assert!(matches!(execution.await, Err(Error::StaleBucketIncarnation { .. })));
let outcome = task.get_outcome().await;
let repaired = outcome
.objects
.iter()
.find(|item| item.identity.object == "b")
.expect("b outcome");
assert_eq!(repaired.identity.bucket_incarnation_id, Some(original));
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
}
#[tokio::test(start_paused = true)]
async fn stale_retry_does_not_start_other_queued_retries_while_draining() {
let original = Uuid::new_v4();
let mut storage = pages_storage(&[&["a", "c", "b"]]);
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original);
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
fail_once(&storage, "a");
fail_once(&storage, "c");
let storage = Arc::new(storage);
let mut task = bucket_task(storage.clone());
task.bucket_incarnation_id = Some(original);
*task.replacement_execution.write().await =
Some(crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new()));
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "c", "b"]);
let before = storage.bucket_incarnation_reads.load(Ordering::Relaxed);
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4());
assert!(matches!(execution.await, Err(Error::StaleBucketIncarnation { .. })));
assert_eq!(storage.bucket_incarnation_reads.load(Ordering::Relaxed), before + 1);
}
#[tokio::test(start_paused = true)]
async fn slow_retries_keep_inflight_work_bounded() {
let names: Vec<String> = (0..10).map(|index| format!("retry-{index}")).collect();
let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect();
page.push(heal_item("slow"));
let mut storage = MockStorage {
retry_test_pages: Some(vec![page]),
..Default::default()
};
storage.retry_test_delays.insert("slow".to_string(), Duration::from_secs(31));
for name in &names {
fail_once(&storage, name);
storage
.retry_test_attempt_delays
.lock()
.expect("attempt delays")
.insert(name.clone(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)]));
}
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 18);
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 10);
task.cancel_token.cancel();
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
tokio::time::advance(Duration::from_secs(60)).await;
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 18);
}
#[tokio::test(start_paused = true)]
async fn inline_retry_is_not_starved_by_slow_deferred_retries() {
let names: Vec<_> = (0..256).map(|index| format!("slow-{index:03}")).collect();
let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect();
page.push(heal_item("inline"));
let storage = MockStorage {
retry_test_pages: Some(vec![page]),
..Default::default()
};
for name in &names {
fail_once(&storage, name);
storage
.retry_test_attempt_delays
.lock()
.expect("delays")
.insert(name.clone(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)]));
}
fail_once(&storage, "inline");
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
assert_eq!(
storage
.heal_object_calls
.lock()
.expect("calls")
.iter()
.filter(|name| name.as_str() == "inline")
.count(),
2,
"the overflow identity must be retried before slow work occupies every slot"
);
task.cancel_token.cancel();
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
}
#[tokio::test(start_paused = true)]
async fn concurrent_retry_overflow_preserves_every_attempted_identity() {
assert_concurrent_retry_overflow_preserves_identities(true).await;
}
#[tokio::test(start_paused = true)]
async fn concurrent_retry_overflow_completes_every_attempted_identity() {
assert_concurrent_retry_overflow_preserves_identities(false).await;
}
async fn assert_concurrent_retry_overflow_preserves_identities(cancel: bool) {
let budget = super::super::heal_bucket::deferred_payload_budget_for_test();
let mut lengths = vec![100; 7];
let mut remaining = budget - 300 - 700;
while remaining > 1_000 {
lengths.push(1_000);
remaining -= 1_000;
}
if remaining > 0 && remaining < 4 {
*lengths.last_mut().expect("previous key") -= 4 - remaining;
remaining = 4;
}
if remaining != 0 {
lengths.push(remaining);
}
assert!(lengths.len() < 256);
let names: Vec<_> = lengths
.into_iter()
.enumerate()
.map(|(index, length)| format!("{index:04}{}", "x".repeat(length - 4)))
.collect();
let extra = format!("extra{}", "y".repeat(895));
let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect();
page.push(heal_item(&extra));
let storage = MockStorage {
retry_test_pages: Some(vec![page]),
..Default::default()
};
for (index, name) in names.iter().enumerate() {
storage.heal_object_outcomes.lock().expect("outcomes").insert(
name.clone(),
VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]),
);
let delays = if index < 7 {
vec![Duration::ZERO, Duration::from_millis(1200)]
} else if index == 7 {
vec![Duration::from_millis(1500)]
} else {
Vec::new()
};
storage
.retry_test_attempt_delays
.lock()
.expect("delays")
.insert(name.clone(), delays.into());
}
storage.heal_object_outcomes.lock().expect("outcomes").insert(
extra.clone(),
VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]),
);
storage.retry_test_attempt_delays.lock().expect("delays").insert(
extra,
VecDeque::from([
Duration::from_millis(1100),
if cancel {
Duration::from_secs(31)
} else {
Duration::from_secs(2)
},
]),
);
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(5), &mut execution).await.is_err());
if cancel {
task.cancel_token.cancel();
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
} else {
execution.await.expect("all attempted identities finish");
}
assert_eq!(
task.get_outcome().await.counters.processed,
u64::try_from(names.len() + 1).expect("count")
);
}
#[tokio::test(start_paused = true)]
async fn due_retry_does_not_overlap_another_version_of_same_object() {
let mut first = heal_item("a");
first.version_id = Some("first".to_string());
let mut second = heal_item("a");
second.version_id = Some("second".to_string());
let mut storage = MockStorage {
retry_test_pages: Some(vec![vec![first, second]]),
..Default::default()
};
storage.retry_test_delays.insert("a".to_string(), Duration::from_secs(31));
fail_once(&storage, "a");
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(tokio::time::timeout(Duration::from_secs(45), &mut execution).await.is_err());
assert_eq!(
storage.heal_object_calls.lock().expect("calls").as_slice(),
["a", "a"],
"a retry must not overlap an in-flight version of the same object"
);
}
#[tokio::test(start_paused = true)]
async fn retry_uses_remaining_time_before_age_limit() {
let mut storage = pages_storage(&[&["a", "b"]]);
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(27));
storage.heal_object_outcomes.lock().expect("outcomes").insert(
"a".to_string(),
VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]),
);
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
task.execute().await.expect("the last retry can run before the age limit");
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a", "a"]);
assert_eq!(task.get_outcome().await.counters.failed, 0);
}
#[tokio::test(start_paused = true)]
async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
let mut first = heal_item("a");
first.version_id = Some("first".to_string());
let mut second = heal_item("a");
second.version_id = Some("second".to_string());
let mut storage = MockStorage {
retry_test_pages: Some(vec![vec![first, second]]),
..Default::default()
};
storage.retry_test_delays.insert("a".to_string(), Duration::from_secs(31));
fail_once(&storage, "a");
let storage = Arc::new(storage);
let task = bucket_task(storage.clone());
task.execute().await.expect_err("aged pending responsibility is not success");
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "a"]);
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.processed, 2);
assert_eq!(outcome.counters.attempt_failures, 1);
@@ -310,7 +767,7 @@ async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
outcome
.objects
.iter()
.find(|item| item.identity.object == "a")
.find(|item| item.identity.version_id.as_deref() == Some("first"))
.expect("a outcome")
.disposition,
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
@@ -345,13 +802,26 @@ async fn cancellation_drains_owned_retries_once() {
#[tokio::test(start_paused = true)]
async fn deadline_drains_owned_retries_without_false_completion() {
let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
let incarnation = Uuid::new_v4();
let storage = pages_storage(&[&["a"], &["b"]]);
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(incarnation);
let storage = Arc::new(storage);
fail_once(&storage, "a");
let mut task = bucket_task(storage.clone());
task.options.timeout = Some(Duration::from_secs(1));
assert!(matches!(task.execute().await, Err(Error::TaskTimeout)));
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.processed, 2);
assert_eq!(
outcome
.objects
.iter()
.find(|item| item.identity.object == "a")
.expect("a")
.identity
.bucket_incarnation_id,
Some(incarnation)
);
assert_eq!(
outcome
.objects
@@ -41,7 +41,7 @@ async fn count_cap_and_next_item_preserve_ownership() {
assert_eq!(window.objects.len(), MAX_DEFERRED_OBJECTS);
assert!(window.bytes <= MAX_DEFERRED_BYTES);
assert!(!window.can_advance(1));
assert!(window.pop_due().is_some());
assert!(window.pop_due(&HashSet::new()).is_some());
assert!(window.push(rejected).is_ok());
}
@@ -56,20 +56,35 @@ async fn byte_cap_counts_key_version_and_reserved_slots() {
assert_eq!(window.bytes, MAX_DEFERRED_BYTES);
assert!(!window.can_advance(1));
assert!(window.push(item("x".to_string(), None)).is_err());
assert!(window.pop_due().is_some());
assert!(window.pop_due(&HashSet::new()).is_some());
assert_eq!(window.bytes, MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>());
assert!(window.push(item("x".to_string(), None)).is_ok());
}
#[tokio::test(start_paused = true)]
async fn due_retry_excludes_inflight_keys_without_hiding_other_keys() {
let mut window = DeferredWindow::default();
assert!(window.push(item("same".to_owned(), Some("v1".to_owned()))).is_ok());
assert!(window.push(item("other".to_owned(), None)).is_ok());
let mut in_flight = HashSet::from(["same".to_owned()]);
assert_eq!(window.pop_due(&in_flight).expect("unblocked retry").name, "other");
assert!(window.next_due(&in_flight).is_none());
assert!(window.pop_due(&in_flight).is_none());
in_flight.clear();
assert!(window.next_due(&in_flight).is_some());
assert_eq!(window.pop_due(&in_flight).expect("released retry").name, "same");
}
#[tokio::test(start_paused = true)]
async fn retry_age_caps_due_time_and_is_not_reset_by_rescheduling() {
let mut entry = item("a".to_string(), None);
entry.defer(Duration::from_secs(2));
let first = entry.first_failure.expect("first failure");
assert_eq!(entry.due, first + Duration::from_secs(2));
tokio::time::advance(Duration::from_secs(29)).await;
entry.defer(Duration::from_secs(8));
assert_eq!(entry.first_failure, Some(first));
assert_eq!(entry.due, first + MAX_DEFERRED_AGE);
assert_eq!(entry.due, first + Duration::from_millis(29_500));
assert!(!entry.expired());
tokio::time::advance(Duration::from_secs(1)).await;
assert!(entry.expired());
+18 -1
View File
@@ -194,13 +194,21 @@ async fn empty_replacement_with_committed_delete(
mod serial_tests {
use super::*;
use storage_api::integration::InstanceContext;
async fn bucket_directory_loss_task(cluster: bool, missing: usize, dry_run: bool, source: HealRequestSource) {
let temp = tempfile::tempdir().expect("isolated bucket recovery fixture");
let env = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(temp.path())
.instance_context(Arc::new(InstanceContext::new()))
.build()
.await;
let shutdown = env
.ecstore
.background_cancel_token()
.expect("fixture store must own a shutdown token");
assert!(!shutdown.is_cancelled(), "a new fixture must not inherit a previous store's shutdown");
eprintln!("bucket recovery: environment ready");
let bucket = "task-directory-recovery";
env.make_bucket(bucket, false).await;
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
@@ -213,6 +221,7 @@ mod serial_tests {
.await
.expect("remove only an empty bucket directory");
}
eprintln!("bucket recovery: directories removed");
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).exists()).count(), 4 - missing);
let mut request = HealRequest::new(
if cluster {
@@ -237,6 +246,7 @@ mod serial_tests {
}
let task = HealTask::from_request(request, storage.clone());
let result = task.execute().await;
eprintln!("bucket recovery: task returned");
if missing == 4 || source != HealRequestSource::Admin {
assert!(
result.is_err(),
@@ -259,6 +269,7 @@ mod serial_tests {
};
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).is_dir()).count(), remaining);
assert_eq!(storage.admit_bucket_incarnation(bucket).await.expect("retained metadata"), expected);
shutdown.cancel();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
@@ -267,7 +278,13 @@ mod serial_tests {
for cluster in [false, true] {
// Two surviving directories satisfy ordinary reads; one does not.
for missing in [2, 3] {
bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin).await;
eprintln!("bucket recovery: cluster={cluster}, missing={missing}");
tokio::time::timeout(
Duration::from_secs(60),
bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin),
)
.await
.unwrap_or_else(|_| panic!("bucket recovery did not finish: cluster={cluster}, missing={missing}"));
}
}
}
+1
View File
@@ -30,6 +30,7 @@ pub(crate) mod integration {
ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCommitGuard, WriteCompletion,
};
pub(crate) use rustfs_ecstore::api::rebalance::{RebalStatus, RebalanceMeta, RebalanceStats};
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
pub(crate) use rustfs_ecstore::api::storage::ECStore;
pub(crate) use rustfs_storage_api::BucketOperations;
pub(crate) use rustfs_storage_api::MakeBucketOptions;
+2 -32
View File
@@ -16,10 +16,9 @@ use super::{
transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
};
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
use openidconnect::{
AsyncHttpClient, ClaimsVerificationError, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
};
use serde::Deserialize;
use std::collections::HashMap;
@@ -213,35 +212,6 @@ impl DiscoveredProviderMetadata {
}
}
pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
token: &'a CoreIdToken,
verifier: &CoreIdTokenVerifier<'_>,
nonce_verifier: N,
) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
CoreJwsSigningAlgorithm::HmacSha256 => 32,
CoreJwsSigningAlgorithm::HmacSha384 => 48,
CoreJwsSigningAlgorithm::HmacSha512 => 64,
_ => return token.claims(verifier, nonce_verifier),
};
// RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
// openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
let invalid_length = || {
ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
"invalid HMAC signature length".to_owned(),
))
};
let compact = token.to_string();
let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
let signature = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(signature)
.map_err(|_| invalid_length())?;
if signature.len() != expected_length {
return Err(invalid_length());
}
token.claims(verifier, nonce_verifier)
}
// This adapter is used only for discovery/JWKS fetches, never token exchange.
struct JwksAcceptClient<'a> {
inner: &'a ReqwestHttpClient,
+5 -11
View File
@@ -242,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
let now = time::OffsetDateTime::now_utc().unix_timestamp();
let payload = serde_json::json!({
"iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
"iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
"iat": now, "exp": now + 300,
});
for (algorithm, name, signature_length) in [
@@ -275,28 +275,22 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
.expect("parse configured provider metadata");
let verifier = metadata.verifier(&config);
assert!(
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
"valid {name} token must pass for workload={workload}"
);
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
let malformed: CoreIdToken = format!("{signing_input}.{signature}")
.parse()
.expect("invalid signature length must remain parseable before verification");
.expect("invalid signature length must still reach cryptographic verification");
assert!(
verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
"{name} signature length {length} must return an error for workload={workload}"
);
assert!(
verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
"{name} signature length {length} must return an error with a nonce for workload={workload}"
);
}
assert!(
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
"invalid inputs must not prevent later valid {name} verification"
);
}
+7 -5
View File
@@ -16,7 +16,7 @@ use super::{
OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
SourcedOidcProviderConfig, load_oidc_config_snapshot,
},
provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
provider::{ProviderRuntime, discover_provider_for_recovery},
state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
};
@@ -535,7 +535,7 @@ impl OidcSys {
let verifier = client
.id_token_verifier()
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
if let Err(e) = verified {
let verification_error = e.to_string();
warn!(
@@ -583,7 +583,8 @@ impl OidcSys {
let verifier = client
.id_token_verifier()
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
id_token
.claims(&verifier, &Nonce::new(session.nonce.clone()))
.map_err(|retry_err| {
error!(
event = EVENT_OIDC_DIAGNOSTICS,
@@ -766,7 +767,7 @@ impl OidcSys {
.metadata
.verifier(&config)
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
state = self
.provider_runtime
.refresh_provider_state(&provider_id, &config)
@@ -779,7 +780,8 @@ impl OidcSys {
.metadata
.verifier(&config)
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
id_token
.claims(&verifier, |_: Option<&Nonce>| Ok(()))
.map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
}
+62
View File
@@ -382,6 +382,42 @@ impl LockClient for LocalClient {
}
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let shard = self.get_shard(lock_id);
let mut guards = shard.write().await;
let Some(entry) = guards.get_mut(lock_id) else {
return Ok(false);
};
if entry.is_expired() {
return Ok(false);
}
Ok(entry.guard.downgrade_committed_tail())
}
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let shard = self.get_shard(lock_id);
let mut guards = shard.write().await;
let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
if entry.is_expired() {
return Ok(false);
}
Ok(entry.guard.try_upgrade_committed_tail())
}
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
self.refresh(lock_id).await
}
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
let shard = self.get_shard(lock_id);
let guards = shard.read().await;
let Some(entry) = guards.get(lock_id) else { return Ok(None) };
if entry.is_expired() {
return Ok(None);
}
Ok(entry.guard.waiting_writer())
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
self.release(lock_id).await
}
@@ -547,6 +583,32 @@ mod tests {
assert!(!response.success);
}
#[tokio::test]
async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
assert!(client.acquire_lock(&first).await.expect("initial lock").success);
assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
assert!(
client
.downgrade_committed_tail(&first.lock_id)
.await
.expect("retry downgrade")
);
let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
.with_acquire_timeout(Duration::from_millis(500));
assert!(client.acquire_lock(&reader).await.expect("reader").success);
let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
assert!(client.release(&first.lock_id).await.expect("release tail"));
assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
assert!(client.release(&reader.lock_id).await.expect("release reader"));
assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
assert!(client.release(&writer.lock_id).await.expect("release writer"));
assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
}
#[tokio::test]
async fn acquire_rejects_a_lock_id_for_another_resource() {
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
+27
View File
@@ -20,6 +20,12 @@ use async_trait::async_trait;
use futures::future::join_all;
use std::sync::Arc;
/// Marker exchanged in a refresh request and response for a committed-tail conversion.
pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
/// Lock client trait
#[async_trait]
pub trait LockClient: Send + Sync + std::fmt::Debug {
@@ -48,6 +54,27 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
/// Refresh lock
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
/// Convert an exclusive lease to a reader-compatible committed-tail lease.
/// Unsupported peers retain their original lease and report `false`.
async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
Ok(false)
}
/// Try to restore exclusive ownership, retaining the writer fence on failure.
async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
Ok(false)
}
/// Check support without changing the lease mode.
async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
Ok(false)
}
/// Observe foreground writer contention, or return `None` when unsupported.
async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
Ok(None)
}
/// Force release lock
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
+75 -1
View File
@@ -262,6 +262,8 @@ pub struct DistributedLockGuard {
/// All underlying leases that should be refreshed and released with the guard.
entries: Vec<HeldLockEntry>,
lock_type: LockType,
/// Shared and exclusive quorums must intersect before a tail may convert.
committed_tail_quorums_intersect: bool,
/// If true, Drop will not try to release (used if user manually released).
disarmed: bool,
/// Background lease task for renewable leases.
@@ -277,7 +279,13 @@ impl DistributedLockGuard {
/// - `entries` is the full list of underlying leases that should be refreshed and released
/// with this guard.
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
fn new(
lock_id: LockId,
entries: Vec<HeldLockEntry>,
lock_type: LockType,
heartbeat: LockHeartbeatConfig,
committed_tail_quorums_intersect: bool,
) -> Self {
record_lock_held_acquire(lock_type);
let lock_lost = Arc::new(LockLostSignal::default());
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
@@ -296,6 +304,7 @@ impl DistributedLockGuard {
lock_id,
entries,
lock_type,
committed_tail_quorums_intersect,
disarmed: false,
refresh_task,
lock_lost,
@@ -412,6 +421,70 @@ impl DistributedLockGuard {
&self.lock_id
}
/// Convert retained peer leases without ever releasing writer exclusion.
/// A peer without conversion support keeps its exclusive lease. Partial
/// conversion is safe: both lease modes continue to exclude writers.
pub async fn downgrade_committed_tail(&mut self) -> bool {
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
{
return false;
}
let results = futures::future::join_all(
self.entries
.iter()
.map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
)
.await;
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
}
/// Require every retained lease to confirm exclusivity before committing.
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
return false;
}
let results = futures::future::join_all(
self.entries
.iter()
.map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
)
.await;
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
}
pub async fn supports_heal_lock_transition(&self) -> bool {
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
{
return false;
}
let results = futures::future::join_all(
self.entries
.iter()
.map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
)
.await;
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
}
/// Observe contention on every held lease. Unknown peers prevent a false idle result.
pub async fn waiting_writer(&self) -> Option<bool> {
if self.disarmed || self.is_lock_lost() {
return None;
}
let results =
futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
if self.is_lock_lost() {
return None;
}
if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
return Some(true);
}
results
.iter()
.all(|result| matches!(result, Ok(Some(false))))
.then_some(false)
}
/// Actual granted peer identities, which may differ from the logical guard ID.
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
self.entries.iter().map(|entry| &entry.lock_id)
@@ -599,6 +672,7 @@ impl DistributedLock {
individual_locks,
request.lock_type,
heartbeat,
self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
)))
} else {
// Check if it's a timeout or quorum failure
+109 -5
View File
@@ -19,8 +19,8 @@ use crate::fast_lock::{
use rustfs_io_metrics::{
record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Weak};
/// Global counter for guard IDs to prevent double-release
static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1);
@@ -39,10 +39,25 @@ pub struct FastLockGuard {
shard: Option<Arc<LockShard>>, // None when locks are disabled
released: bool,
disabled: bool, // True when locks are disabled globally
committed_tail: bool,
sealed_for_commit: bool,
/// Unique ID for this guard instance to prevent double-release
guard_id: u64,
}
/// A non-owning check of one local acquisition, including forced release.
#[derive(Clone)]
pub struct FastLockGuardProbe {
shard: Weak<LockShard>,
guard_id: u64,
}
impl FastLockGuardProbe {
pub fn is_lost(&self) -> bool {
self.shard.upgrade().is_none_or(|shard| !shard.is_guard_active(self.guard_id))
}
}
impl FastLockGuard {
pub(crate) fn new(key: ObjectKey, mode: LockMode, owner: Arc<str>, shard: Arc<LockShard>) -> Self {
let guard_id = GUARD_ID_COUNTER.fetch_add(1, Ordering::Relaxed);
@@ -54,6 +69,8 @@ impl FastLockGuard {
shard: Some(shard),
released: false,
disabled: false,
committed_tail: false,
sealed_for_commit: false,
guard_id,
}
}
@@ -68,6 +85,8 @@ impl FastLockGuard {
shard: None,
released: false,
disabled: true,
committed_tail: false,
sealed_for_commit: false,
guard_id,
}
}
@@ -87,6 +106,83 @@ impl FastLockGuard {
&self.owner
}
/// The physical acquisition was force-released before registration.
pub(crate) fn abandon_unregistered(&mut self) {
if !self.released {
self.released = true;
record_lock_held_release(self.mode);
}
}
/// Keep excluding writers while permitting reads of the committed version.
pub fn downgrade_committed_tail(&mut self) -> bool {
if self.released || self.sealed_for_commit {
return false;
}
if self.committed_tail {
return true;
}
if self.mode != LockMode::Exclusive {
return false;
}
if self.disabled {
self.mode = LockMode::Shared;
self.committed_tail = true;
return true;
}
let Some(shard) = &self.shard else { return false };
if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
return false;
}
record_lock_held_release(LockMode::Exclusive);
record_lock_held_acquire(LockMode::Shared);
self.mode = LockMode::Shared;
self.committed_tail = true;
true
}
/// Try to restore exclusive ownership without releasing the writer fence.
pub fn try_upgrade_committed_tail(&mut self) -> bool {
if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
return false;
}
if !self.disabled {
let Some(shard) = &self.shard else { return false };
if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
return false;
}
}
if self.committed_tail {
record_lock_held_release(LockMode::Shared);
record_lock_held_acquire(LockMode::Exclusive);
}
self.mode = LockMode::Exclusive;
self.committed_tail = false;
self.sealed_for_commit = true;
true
}
/// Observe writers waiting on this guard's exact object key.
pub fn waiting_writer(&self) -> Option<bool> {
if self.released {
return None;
}
if self.disabled {
return Some(false);
}
self.shard
.as_ref()?
.waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
}
fn release_physical(&self, shard: &LockShard) -> bool {
if self.committed_tail {
shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
} else {
shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
}
}
/// Manually release the lock early
///
/// Returns true if the lock was successfully released, false if it was
@@ -106,7 +202,7 @@ impl FastLockGuard {
}
if let Some(shard) = &self.shard {
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
let success = self.release_physical(shard);
if success {
self.released = true;
record_lock_held_release(self.mode);
@@ -138,9 +234,17 @@ impl FastLockGuard {
);
}
/// Check if the lock has been released
/// Check if the lock has been released, including by an administrator.
pub fn is_released(&self) -> bool {
self.released
self.released || self.shard.as_ref().is_some_and(|shard| !shard.is_guard_active(self.guard_id))
}
/// Observe this acquisition without extending the lock's lifetime.
pub fn release_probe(&self) -> Option<FastLockGuardProbe> {
self.shard.as_ref().map(|shard| FastLockGuardProbe {
shard: Arc::downgrade(shard),
guard_id: self.guard_id,
})
}
/// Check if this guard represents a disabled lock
@@ -169,7 +273,7 @@ impl Drop for FastLockGuard {
fn drop(&mut self) {
if let Some(shard) = &self.shard {
if !self.released && !self.disabled {
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
let success = self.release_physical(shard);
if !success {
// For high-concurrency scenarios, this is likely due to:
// 1. Lock was already released by another thread
+16 -33
View File
@@ -96,11 +96,14 @@ impl FastObjectLockManager {
/// Acquire object lock
pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
let shard = self.get_shard(&request.key);
match shard.acquire_lock(&request).await {
Ok(()) => {
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
match shard.acquire_guarded(&request).await {
Ok(acquired) => {
let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
// Register guard to prevent premature cleanup
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
guard.abandon_unregistered();
return Err(LockResult::Timeout);
}
Ok(guard)
}
Err(err) => Err(err),
@@ -201,31 +204,15 @@ impl FastObjectLockManager {
let mut all_failed = Vec::new();
let mut guards = Vec::new();
for (shard_id, requests) in shard_groups {
let shard = self.shards[*shard_id].clone();
for (_, requests) in shard_groups {
for request in requests {
let key = request.key.clone();
let owner = request.owner.clone();
let mode = request.mode;
let acquired = if shard.try_fast_path_only(request) {
true
} else {
match shard.acquire_lock(request).await {
Ok(()) => true,
Err(err) => {
all_failed.push((key.clone(), err));
false
}
match self.acquire_lock(request.clone()).await {
Ok(guard) => {
all_successful.push(key);
guards.push(guard);
}
};
if acquired {
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
all_successful.push(key);
guards.push(guard);
Err(err) => all_failed.push((key, err)),
}
}
}
@@ -245,14 +232,10 @@ impl FastObjectLockManager {
let mut acquired_guards = Vec::new();
let mut failed_locks = Vec::new();
'outer: for (shard_id, requests) in shard_groups {
let shard = self.shards[*shard_id].clone();
'outer: for (_, requests) in shard_groups {
for request in requests {
match shard.acquire_lock(request).await {
Ok(()) => {
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
match self.acquire_lock(request.clone()).await {
Ok(guard) => {
acquired_guards.push(guard);
}
Err(err) => {
+1
View File
@@ -94,6 +94,7 @@ impl ObjectLockState {
pub fn reset_for_reuse(&mut self) {
// Reset atomic state
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
self.force_epoch.store(0, Ordering::Relaxed);
// Clear owners
*self.current_owner.write() = None;
+500 -49
View File
@@ -15,6 +15,7 @@
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant, SystemTime};
use tokio::time::timeout;
@@ -30,6 +31,12 @@ struct ActiveGuardInfo {
key: ObjectKey,
mode: LockMode,
owner: Arc<str>,
committed_tail: bool,
}
pub(crate) struct AcquiredLock {
state: Arc<ObjectLockState>,
force_epoch: u64,
}
#[derive(Debug, PartialEq, Eq, Hash)]
@@ -46,6 +53,8 @@ enum TestPausePoint {
BeforeFastWrite,
FastWriteExisting,
SlowLookup,
ForceAfterTailRelease,
ReleaseAfterUnregister,
}
#[cfg(test)]
@@ -69,7 +78,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
/// Lock shard to reduce global contention.
///
/// When both are needed, lock `objects` before a state's owner metadata.
/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
#[derive(Debug)]
pub struct LockShard {
/// Object lock states - using parking_lot for better performance
@@ -133,12 +142,16 @@ impl LockShard {
/// Acquire lock with fast path optimization
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
self.acquire_guarded(request).await.map(|_| ())
}
pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
let start_time = Instant::now();
// Try fast path first
if let Some(_state) = self.try_fast_path(request) {
if let Some(acquired) = self.try_fast_path(request) {
self.metrics.record_fast_path_success();
return Ok(());
return Ok(acquired);
}
// Slow path with waiting
@@ -157,7 +170,7 @@ impl LockShard {
}
/// Try fast path lock acquisition (lock-free when possible)
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
// First try to get existing state without write lock
{
let objects = self.objects.read();
@@ -168,14 +181,16 @@ impl LockShard {
test_pause(TestPausePoint::FastExisting, &request.key);
// Try atomic acquisition
let success = match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
let force_epoch = state.force_epoch.load(Ordering::Acquire);
let success = force_epoch != u64::MAX
&& match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
drop(objects);
if success {
return Some(state);
return Some(AcquiredLock { state, force_epoch });
}
}
}
@@ -195,10 +210,11 @@ impl LockShard {
#[cfg(test)]
test_pause(TestPausePoint::FastWriteExisting, &request.key);
let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
let force_epoch = state.force_epoch.load(Ordering::Acquire);
let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
drop(objects);
if acquired {
return Some(state);
return Some(AcquiredLock { state, force_epoch });
}
} else {
// Create new state from pool and acquire immediately
@@ -206,7 +222,7 @@ impl LockShard {
let state = Arc::new(*state_box);
if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
objects.insert(request.key.clone(), state.clone());
return Some(state);
return Some(AcquiredLock { state, force_epoch: 0 });
}
}
}
@@ -215,7 +231,7 @@ impl LockShard {
}
/// Slow path with async waiting
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
// Use adaptive timeout based on current load and request priority
let adaptive_timeout = self.calculate_adaptive_timeout(request);
let deadline = start_time + adaptive_timeout;
@@ -234,7 +250,7 @@ impl LockShard {
loop {
// Keep the state in the map until its acquisition attempt completes.
let (state, success) = {
let (state, success, force_epoch) = {
let mut objects = self.objects.write();
let state = match objects.get(&request.key) {
Some(state) => state.clone(),
@@ -249,17 +265,19 @@ impl LockShard {
#[cfg(test)]
test_pause(TestPausePoint::SlowLookup, &request.key);
let success = match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
let force_epoch = state.force_epoch.load(Ordering::Acquire);
let success = force_epoch != u64::MAX
&& match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
drop(objects);
(state, success)
(state, success, force_epoch)
};
if success {
self.metrics.record_slow_path_success();
return Ok(());
return Ok(AcquiredLock { state, force_epoch });
}
// Check timeout
@@ -381,14 +399,10 @@ impl LockShard {
/// Release lock with guard ID tracking for double-release prevention
pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
// First, try to remove the guard from active set
let guard_was_active = {
let mut guards = self.active_guards.lock();
guards.remove(&guard_id).is_some()
};
// If guard was not active, this is a double-release attempt
if !guard_was_active {
// Keep force-unlock from releasing this holder and admitting a new
// same-owner holder between guard removal and physical release.
let mut guards = self.active_guards.lock();
if guards.remove(&guard_id).is_none() {
tracing::debug!(
"Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
key,
@@ -399,6 +413,9 @@ impl LockShard {
return false;
}
#[cfg(test)]
test_pause(TestPausePoint::ReleaseAfterUnregister, key);
// Proceed with normal release
let should_cleanup;
let result;
@@ -423,6 +440,8 @@ impl LockShard {
}
}
drop(guards);
if should_cleanup {
self.schedule_cleanup(key.clone());
}
@@ -430,21 +449,110 @@ impl LockShard {
result
}
/// Convert an active exclusive guard without opening a writer gap.
pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
let mut guards = self.active_guards.lock();
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
return false;
}
let objects = self.objects.read();
let Some(state) = objects.get(key) else { return false };
if !state.downgrade_committed_tail(owner) {
return false;
}
info.mode = LockMode::Shared;
info.committed_tail = true;
true
}
pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
let mut guards = self.active_guards.lock();
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
return false;
}
if info.mode == LockMode::Exclusive && !info.committed_tail {
return true;
}
if info.mode != LockMode::Shared || !info.committed_tail {
return false;
}
let objects = self.objects.read();
let Some(state) = objects.get(key) else { return false };
if !state.try_upgrade_committed_tail(owner) {
return false;
}
info.mode = LockMode::Exclusive;
info.committed_tail = false;
true
}
pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
let guards = self.active_guards.lock();
let info = guards.get(&guard_id)?.as_ref()?;
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
return None;
}
let objects = self.objects.read();
let state = objects.get(key)?;
Some(state.atomic_state.writer_observed_during_tail())
}
/// Release a converted guard and its special reader-admission state.
pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
let mut guards = self.active_guards.lock();
let Some(Some(info)) = guards.get(&guard_id) else { return false };
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
return false;
}
let (success, should_cleanup) = {
let objects = self.objects.read();
let Some(state) = objects.get(key) else { return false };
let success = state.release_committed_tail(owner);
(success, success && !state.is_locked() && !state.atomic_state.has_waiters())
};
if success {
guards.remove(&guard_id);
self.metrics.record_release();
}
drop(guards);
if should_cleanup {
self.schedule_cleanup(key.clone());
}
success
}
/// Register a guard to prevent premature cleanup
pub fn register_guard(&self, guard_id: u64) {
self.active_guards.lock().insert(guard_id, None);
}
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
pub(crate) fn register_guard_with_info(
&self,
guard_id: u64,
key: &ObjectKey,
mode: LockMode,
owner: &Arc<str>,
acquired: &AcquiredLock,
) -> bool {
let mut guards = self.active_guards.lock();
let objects = self.objects.read();
if !objects.get(key).is_some_and(|state| {
Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
}) {
return false;
}
guards.insert(
guard_id,
Some(ActiveGuardInfo {
key: key.clone(),
mode,
owner: owner.clone(),
committed_tail: false,
}),
);
true
}
/// Unregister a guard (called when guard is dropped)
@@ -460,9 +568,8 @@ impl LockShard {
guards.len()
}
/// Check if a guard is active (for testing)
#[cfg(test)]
pub fn is_guard_active(&self, guard_id: u64) -> bool {
/// Check whether this exact guard still owns its acquisition.
pub(crate) fn is_guard_active(&self, guard_id: u64) -> bool {
let guards = self.active_guards.lock();
guards.contains_key(&guard_id)
}
@@ -719,27 +826,57 @@ impl LockShard {
/// Returns the number of owners that were released. Used by the admin
/// force-unlock path to clear a stuck resource.
pub fn force_release_all(&self, key: &ObjectKey) -> usize {
let owners_modes: Vec<(Arc<str>, LockMode)> = {
let objects = self.objects.read();
let Some(state) = objects.get(key) else {
return 0;
};
let mut pairs = Vec::new();
if let Some(info) = state.current_owner.read().clone() {
pairs.push((info.owner, LockMode::Exclusive));
}
for entry in state.shared_owners.read().iter() {
pairs.push((entry.owner.clone(), LockMode::Shared));
}
pairs
};
// Serialize tail conversion and prevent new acquisitions while taking
// and releasing the owner snapshot. A new tail must never be mistaken
// for an ordinary shared owner by force unlock.
let mut guards = self.active_guards.lock();
let objects = self.objects.write();
let Some(state) = objects.get(key) else { return 0 };
// Acquisitions completed before this point but not yet registered
// must not later create guards for the released generation.
let _ = state
.force_epoch
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
let tail = guards.iter().find_map(|(&id, info)| {
info.as_ref()
.filter(|info| info.committed_tail && info.key == *key)
.map(|info| (id, info.owner.clone()))
});
let mut released = 0;
for (owner, mode) in owners_modes {
if self.release_lock(key, &owner, mode) {
if let Some((id, owner)) = tail
&& state.release_committed_tail(&owner)
{
guards.remove(&id);
self.metrics.record_release();
released += 1;
}
#[cfg(test)]
test_pause(TestPausePoint::ForceAfterTailRelease, key);
let mut owners_modes = Vec::new();
if let Some(info) = state.current_owner.read().clone() {
owners_modes.push((info.owner, LockMode::Exclusive, 1));
}
for entry in state.shared_owners.read().iter() {
owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
}
for (owner, mode, count) in owners_modes {
let mut owner_released = false;
for _ in 0..count {
let success = match mode {
LockMode::Shared => state.release_shared(&owner),
LockMode::Exclusive => state.release_exclusive(&owner),
};
if success {
self.metrics.record_release();
owner_released = true;
}
}
if owner_released {
released += 1;
}
}
// Old guard drops must not release a later holder with the same owner.
guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
released
}
@@ -979,6 +1116,320 @@ mod tests {
)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
for reader in [false, true] {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-release-gap");
let old = if reader {
manager.acquire_read_lock(key.clone(), "same-owner").await
} else {
manager.acquire_write_lock(key.clone(), "same-owner").await
}
.expect("old holder");
let shard = manager.get_shard(&key).clone();
let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
let old_drop = tokio::task::spawn_blocking(move || drop(old));
entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
let can_force = shard.active_guards.try_lock().is_some();
let mut force = {
let manager = manager.clone();
let key = key.clone();
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
};
let mut new_guard = if can_force {
assert_eq!(force.await.expect("force join"), 1);
let mut guard = manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("new writer");
if reader {
assert!(guard.downgrade_committed_tail());
}
resume.send(()).expect("resume old release");
old_drop.await.expect("old release join");
guard
} else {
assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
resume.send(()).expect("resume old release");
old_drop.await.expect("old release join");
assert_eq!(force.await.expect("force join"), 0);
manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("new writer")
};
if reader && !can_force {
assert!(new_guard.downgrade_committed_tail());
}
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"an old release must not unlock the new writer"
);
drop(new_guard);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn force_unlock_does_not_capture_a_new_committed_tail() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-tail-race");
let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
assert!(old.downgrade_committed_tail());
let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
let force = {
let manager = manager.clone();
let key = key.clone();
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
};
entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
let mut next = {
let manager = manager.clone();
let key = key.clone();
tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
};
assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
resume.send(()).expect("resume force unlock");
assert_eq!(force.await.expect("force unlock task"), 1);
let mut new = next.await.expect("new writer task").expect("new writer");
assert!(new.downgrade_committed_tail());
drop(old);
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"the new committed tail must still exclude writers"
);
}
#[tokio::test]
async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "upgrade-tail");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
let reader = manager
.acquire_read_lock(key.clone(), "reader")
.await
.expect("reader during tail");
assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
.await
.is_err(),
"new readers must not starve a pending upgrade"
);
drop(reader);
assert!(tail.try_upgrade_committed_tail());
assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
assert!(
!tail.downgrade_committed_tail(),
"a delayed downgrade must not reopen reads after upgrade"
);
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
.await
.is_err()
);
drop(tail);
let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
}
#[tokio::test]
async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "brief-writer-observation");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
.await
.is_err()
);
assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
drop(tail);
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
}
#[tokio::test]
async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert!(tail.try_upgrade_committed_tail());
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
.await
.is_err()
);
assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
drop(tail);
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
assert_eq!(next.waiting_writer(), Some(false));
}
#[tokio::test]
async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "waiting-writer-observation");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert_eq!(tail.waiting_writer(), Some(false));
let writer = {
let manager = manager.clone();
let key = key.clone();
tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
};
tokio::time::timeout(Duration::from_secs(2), async {
while tail.waiting_writer() != Some(true) {
tokio::task::yield_now().await;
}
})
.await
.expect("writer becomes observable");
assert_eq!(manager.force_unlock(&key), 1);
assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
}
#[tokio::test]
async fn dropping_a_pending_upgrade_allows_readers_again() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "cancel-upgrade");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
assert!(!tail.try_upgrade_committed_tail());
let mut waiting = {
let manager = manager.clone();
let key = key.clone();
tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
};
assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
drop(tail);
let second = tokio::time::timeout(Duration::from_secs(2), waiting)
.await
.expect("reader wakes after cancellation")
.expect("reader task joins")
.expect("reader obtains lock");
drop((reader, second));
}
#[tokio::test]
async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "upgrade-stale");
let mut tail = manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert_eq!(manager.force_unlock(&key), 1);
let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
assert!(!tail.try_upgrade_committed_tail());
}
#[tokio::test]
async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "late-downgrade");
let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
assert!(guard.try_upgrade_committed_tail());
assert!(!guard.downgrade_committed_tail());
assert!(guard.try_upgrade_committed_tail());
}
#[tokio::test]
async fn force_unlock_old_readers_cannot_release_a_new_tail() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-old-readers");
let old1 = manager
.acquire_read_lock(key.clone(), "same-owner")
.await
.expect("first reader");
let old2 = manager
.acquire_read_lock(key.clone(), "same-owner")
.await
.expect("second reader");
assert!(!old1.is_released());
assert!(!old2.is_released());
assert_eq!(manager.force_unlock(&key), 1);
let mut tail = manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("new writer");
assert!(tail.downgrade_committed_tail());
assert!(old1.is_released());
assert!(old2.is_released());
assert!(!tail.is_released());
drop((old1, old2));
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"old reader drops must not release the new tail"
);
}
#[tokio::test]
async fn forced_unregistered_reader_cannot_release_a_new_tail() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-unregistered-reader");
let shard = manager.get_shard(&key).clone();
let owner: Arc<str> = Arc::from("same-owner");
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
let acquired = shard
.acquire_guarded(&request)
.await
.expect("old reader acquired before registration");
assert_eq!(manager.force_unlock(&key), 1);
let mut tail = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("new writer");
assert!(tail.downgrade_committed_tail());
let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
old.abandon_unregistered();
drop(old);
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"an old reader must not release a new committed tail"
);
}
#[tokio::test]
async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "unrelated-acquisition");
let shard = manager.get_shard(&key).clone();
let owner: Arc<str> = Arc::from("owner");
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
let other = (0..1000)
.map(|index| ObjectKey::new("bucket", format!("other-{index}")))
.find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
.expect("another key on the same shard");
let other_guard = manager
.acquire_write_lock(other.clone(), "other")
.await
.expect("other writer");
assert_eq!(manager.force_unlock(&other), 1);
drop(other_guard);
let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
drop(guard);
assert!(manager.acquire_write_lock(key, "next").await.is_ok());
}
fn start_acquisition(
manager: &Arc<FastObjectLockManager>,
request: ObjectLockRequest,
+180 -8
View File
@@ -23,6 +23,11 @@ use crate::fast_lock::types::{LockMode, LockPriority};
/// Optimized atomic lock state encoding in u64
/// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
const WRITER_FLAG_MASK: u64 = 0x1;
// A committed write tail is a reader-compatible owner that still excludes writers.
const COMMITTED_TAIL_FLAG: u64 = 0x2;
const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
const READERS_SHIFT: u8 = 8;
const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
const READERS_WAITING_SHIFT: u8 = 16;
@@ -31,7 +36,6 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
// Fast path check masks
const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
const COMPLETELY_UNLOCKED: u64 = 0;
/// Fast atomic lock state for single version
@@ -66,8 +70,8 @@ impl AtomicLockState {
let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
match mode {
LockMode::Shared => {
// No writer and no waiting writers
(state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
(state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
&& ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
}
LockMode::Exclusive => {
// Completely unlocked
@@ -83,8 +87,11 @@ impl AtomicLockState {
loop {
let current = self.state.load(Ordering::Acquire);
// Fast path check - cannot acquire if there's a writer or writers waiting
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
// While a committed tail owns a shared slot, let reads through
// even if a subsequent writer is waiting for that tail to drain.
if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
|| ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
{
return false;
}
@@ -137,6 +144,78 @@ impl AtomicLockState {
}
}
/// Atomically retain writer exclusion while allowing reads of a committed tail.
fn downgrade_committed_tail(&self) -> bool {
loop {
let current = self.state.load(Ordering::Acquire);
if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
return false;
}
let next = (current & !WRITER_FLAG_MASK)
| COMMITTED_TAIL_FLAG
| (1 << READERS_SHIFT)
| if current & WRITERS_WAITING_MASK != 0 {
TAIL_WRITER_OBSERVED_FLAG
} else {
0
};
if self
.state
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
self.update_access_time();
return true;
}
}
}
/// Restore exclusive ownership only after every other reader has left.
fn try_upgrade_committed_tail(&self) -> bool {
loop {
let current = self.state.load(Ordering::Acquire);
if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
return false;
}
let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
let next = if readers_drained {
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
| WRITER_FLAG_MASK
| TAIL_TRACK_WRITERS_FLAG
} else {
current | TAIL_UPGRADE_PENDING_FLAG
};
if self
.state
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
self.update_access_time();
return readers_drained;
}
}
}
/// Release the committed tail's own shared slot and restore ordinary fairness.
fn release_committed_tail(&self) -> bool {
loop {
let current = self.state.load(Ordering::Acquire);
if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
return false;
}
let next =
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
if self
.state
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
self.update_access_time();
return true;
}
}
}
/// Release shared lock
pub fn release_shared(&self) -> bool {
loop {
@@ -169,7 +248,7 @@ impl AtomicLockState {
return false; // No exclusive lock to release
}
let new_state = current & !WRITER_FLAG_MASK;
let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
if self
.state
@@ -236,7 +315,10 @@ impl AtomicLockState {
return false; // Max waiting writers
}
let new_state = current + (1 << WRITERS_WAITING_SHIFT);
let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
new_state |= TAIL_WRITER_OBSERVED_FLAG;
}
if self
.state
@@ -321,10 +403,15 @@ impl AtomicLockState {
}
#[cfg(test)]
pub fn writers_waiting_count(&self) -> u16 {
pub(crate) fn writers_waiting_count(&self) -> u16 {
let state = self.state.load(Ordering::Acquire);
self.writers_waiting(state)
}
pub(crate) fn writer_observed_during_tail(&self) -> bool {
let state = self.state.load(Ordering::Acquire);
self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
}
}
/// Object lock state with version support - optimized memory layout
@@ -334,6 +421,8 @@ pub struct ObjectLockState {
// First cache line: Most frequently accessed data
/// Atomic state for fast operations
pub atomic_state: AtomicLockState,
/// Changes whenever an administrator force-releases this state.
pub(crate) force_epoch: AtomicU64,
// Second cache line: Notification mechanisms
/// Notification for readers (traditional)
@@ -377,6 +466,7 @@ impl ObjectLockState {
pub fn new() -> Self {
Self {
atomic_state: AtomicLockState::new(),
force_epoch: AtomicU64::new(0),
read_notify: Notify::new(),
write_notify: Notify::new(),
optimized_notify: OptimizedNotify::new(),
@@ -425,6 +515,70 @@ impl ObjectLockState {
true
}
/// Convert the current exclusive owner into the committed-tail owner.
pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
let mut current = self.current_owner.write();
let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
return false;
};
let mut shared = self.shared_owners.write();
if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
return false;
}
shared.push(SharedOwnerEntry {
owner: info.owner,
count: 1,
acquired_at: info.acquired_at,
lock_timeout: info.lock_timeout,
});
*current = None;
drop(shared);
drop(current);
self.optimized_notify.notify_readers();
true
}
pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
let mut current = self.current_owner.write();
let mut shared = self.shared_owners.write();
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
return false;
};
if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
return false;
}
let info = shared.remove(pos);
*current = Some(ExclusiveOwnerInfo {
owner: info.owner,
acquired_at: info.acquired_at,
lock_timeout: info.lock_timeout,
});
true
}
/// Release the committed-tail owner's shared slot without releasing readers.
pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
let mut shared = self.shared_owners.write();
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
return false;
};
if !self.atomic_state.release_committed_tail() {
return false;
}
if shared[pos].count > 1 {
shared[pos].count -= 1;
} else {
shared.remove(pos);
}
let no_readers = shared.is_empty();
drop(shared);
if no_readers {
self.optimized_notify.notify_writer();
}
self.optimized_notify.notify_readers();
true
}
/// Release shared lock
pub fn release_shared(&self, owner: &Arc<str>) -> bool {
let mut shared = self.shared_owners.write();
@@ -555,6 +709,24 @@ mod tests {
assert!(!state.release_exclusive());
}
#[test]
fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
let state = AtomicLockState::new();
assert!(state.try_acquire_exclusive());
assert!(state.inc_writers_waiting());
assert!(state.downgrade_committed_tail());
assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
assert!(state.release_committed_tail());
assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
assert!(state.release_shared());
assert!(state.try_acquire_exclusive());
assert!(state.release_exclusive());
state.dec_writers_waiting();
assert!(state.is_free());
}
// Regression for the waiter-preserving exclusive CAS.
//
// The acquisition CAS used to demand a fully-zero state word, which
+61
View File
@@ -111,6 +111,67 @@ mod fast_lock_tests {
assert!(guard.is_released(), "Guard should be marked as released");
}
#[tokio::test]
async fn force_unlock_old_tail_cannot_release_new_tail() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "force-tail-generation");
let owner: Arc<str> = Arc::from("same-owner");
let mut old = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("old writer");
assert!(old.downgrade_committed_tail());
assert_eq!(manager.force_unlock(&key), 1);
let mut new = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("new writer");
assert!(new.downgrade_committed_tail());
drop(old);
let attempt =
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
.await;
assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
drop(new);
}
#[tokio::test]
async fn committed_tail_allows_reader_while_writer_waits() {
let manager = Arc::new(create_test_manager());
let key = ObjectKey::new("test-bucket", "tail-handoff");
let mut tail = manager
.acquire_write_lock(key.clone(), Arc::from("first-writer"))
.await
.expect("initial writer");
let writer = tokio::spawn({
let manager = Arc::clone(&manager);
let key = key.clone();
async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
});
tokio::time::timeout(Duration::from_secs(2), async {
while manager.current_lock_counts().1 == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("second writer should be queued");
assert!(tail.downgrade_committed_tail());
let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
.await
.expect("reader should not wait behind committed tail")
.expect("reader should acquire");
assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
drop(tail);
assert!(!writer.is_finished(), "reader still excludes the next writer");
drop(reader);
let writer = tokio::time::timeout(Duration::from_secs(2), writer)
.await
.expect("writer should resume")
.expect("writer task should join")
.expect("writer should acquire");
drop(writer);
}
#[tokio::test]
async fn current_lock_counts_report_live_holders_and_waiters() {
let manager = Arc::new(create_test_manager());
+32
View File
@@ -107,6 +107,38 @@ impl NamespaceLockGuard {
}
}
/// Permit reads while continuing to exclude competing writers.
pub async fn downgrade_committed_tail(&mut self) -> bool {
match self {
Self::Standard(guard) => guard.downgrade_committed_tail().await,
Self::Fast(guard) => guard.downgrade_committed_tail(),
}
}
/// Try to restore exclusive access while retaining the writer fence.
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
match self {
Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
Self::Fast(guard) => guard.try_upgrade_committed_tail(),
}
}
/// Check whether the held leases can safely use heal lock transitions.
pub async fn supports_heal_lock_transition(&self) -> bool {
match self {
Self::Standard(guard) => guard.supports_heal_lock_transition().await,
Self::Fast(_) => true,
}
}
/// Observe whether a writer is waiting on this object's held leases.
pub async fn waiting_writer(&self) -> Option<bool> {
match self {
Self::Standard(guard) => guard.waiting_writer().await,
Self::Fast(guard) => guard.waiting_writer(),
}
}
/// Manually release the lock early
pub fn release(&mut self) -> bool {
match self {
+317
View File
@@ -154,6 +154,43 @@ impl crate::client::LockClient for DelayedClient {
}
}
#[derive(Debug)]
struct PendingTailClient(LocalClient);
#[async_trait::async_trait]
impl crate::client::LockClient for PendingTailClient {
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
self.0.acquire_lock(request).await
}
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
self.0.release(lock_id).await
}
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
self.0.refresh(lock_id).await
}
async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
std::future::pending().await
}
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
self.0.force_release(lock_id).await
}
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
self.0.check_status(lock_id).await
}
async fn get_stats(&self) -> crate::Result<LockStats> {
self.0.get_stats().await
}
async fn close(&self) -> crate::Result<()> {
self.0.close().await
}
async fn is_online(&self) -> bool {
self.0.is_online().await
}
async fn is_local(&self) -> bool {
self.0.is_local().await
}
}
#[derive(Debug)]
struct FlakyAcquireClient {
inner: LocalClient,
@@ -579,6 +616,286 @@ async fn test_namespace_lock_guard_lock_id() {
}
}
#[tokio::test]
async fn tail_conversion_rejects_non_intersecting_quorums() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
let resource = ObjectKey::new("bucket", "non-intersecting-tail");
let mut tail = lock
.get_write_lock(resource, "writer", Duration::from_secs(2))
.await
.expect("write lease");
assert!(
!tail.downgrade_committed_tail().await,
"conversion requires intersecting read and write quorums"
);
assert!(tail.release());
}
#[tokio::test]
async fn partial_tail_conversion_keeps_writers_excluded() {
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
for index in 0..4 {
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
if index < 2 {
clients.push(inner);
} else {
clients.push(Arc::new(DelayedClient {
inner,
delay: Duration::ZERO,
}));
}
}
let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
let resource = ObjectKey::new("bucket", "partial-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"mixed leases still exclude writers"
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "second", Duration::from_secs(3))
.await
.expect("writer after release");
assert!(writer.release());
}
#[tokio::test]
async fn cancelled_tail_conversion_keeps_writers_excluded() {
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
let resource = ObjectKey::new("bucket", "cancel-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(
tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
.await
.is_err()
);
let mut reader = lock
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
.await
.expect("other peers converted before cancellation");
assert!(reader.release());
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"cancelled conversion must still exclude writers"
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "second", Duration::from_secs(3))
.await
.expect("writer after release");
assert!(writer.release());
}
#[tokio::test]
async fn unsupported_peer_keeps_exclusive_tail_lease() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| {
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
Arc::new(DelayedClient {
inner,
delay: Duration::ZERO,
}) as Arc<dyn crate::client::LockClient>
})
.collect();
let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
let resource = ObjectKey::new("bucket", "old-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
assert!(
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
.await
.is_err()
);
assert!(
lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
.await
.is_err()
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "writer", Duration::from_secs(3))
.await
.expect("writer after tail");
assert!(writer.release());
}
#[tokio::test]
async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
let resource = ObjectKey::new("bucket", "committed-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
let mut reader = lock
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
.await
.expect("committed reader");
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"tail and reader must exclude writer"
);
assert!(tail.release());
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"reader still excludes writer"
);
assert!(reader.release());
let mut writer = lock
.get_write_lock(resource, "second", Duration::from_secs(3))
.await
.expect("writer after drain");
assert!(writer.release());
}
#[tokio::test]
async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
let resource = ObjectKey::new("bucket", "tail-upgrade");
let mut tail = lock
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
.await
.expect("write lock");
assert!(tail.supports_heal_lock_transition().await);
assert!(tail.downgrade_committed_tail().await);
let mut reader = lock
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
.await
.expect("reader");
assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
assert!(reader.release());
tokio::time::timeout(Duration::from_secs(2), async {
while !tail.try_upgrade_committed_tail().await {
tokio::task::yield_now().await;
}
})
.await
.expect("retry must finish partial upgrades after asynchronous reader release");
assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
assert!(
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
.await
.is_err()
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "writer", Duration::from_secs(2))
.await
.expect("writer after release");
assert!(writer.release());
}
#[tokio::test]
async fn distributed_tail_observes_a_waiting_writer() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
let resource = ObjectKey::new("bucket", "writer-observation");
let mut tail = lock
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
.await
.expect("heal lock");
assert!(tail.downgrade_committed_tail().await);
assert_eq!(tail.waiting_writer().await, Some(false));
let writer = {
let lock = Arc::clone(&lock);
let resource = resource.clone();
tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
};
tokio::time::timeout(Duration::from_secs(2), async {
while tail.waiting_writer().await != Some(true) {
tokio::task::yield_now().await;
}
})
.await
.expect("waiting writer becomes visible on a held peer");
assert!(tail.release());
let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
assert!(guard.release());
}
#[tokio::test]
async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
use crate::client::LockClient;
let clients: Vec<Arc<LocalClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
.collect();
let lock = NamespaceLock::with_clients(
"partial-upgrade-writer".to_owned(),
clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
);
let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
let mut tail = lock
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
.await
.expect("heal lock");
let ids = match &tail {
NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
};
let mut holders = Vec::new();
for client in &clients {
for id in &ids {
if client.check_status(id).await.expect("peer status").is_some() {
holders.push(Arc::clone(client));
break;
}
}
}
assert!(holders.len() >= 2, "write quorum must hold multiple peers");
assert!(tail.downgrade_committed_tail().await);
let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
assert_eq!(
tail.waiting_writer().await,
Some(true),
"partial upgrade must retain the earlier contention"
);
assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
assert!(tail.release());
}
#[tokio::test]
async fn test_namespace_lock_distributed_multi_node_simulation() {
// Simulate a 3-node distributed environment where each node has its own lock backend
File diff suppressed because it is too large Load Diff
+140
View File
@@ -1092,6 +1092,56 @@ impl_canonical_mutation_body!(
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
// the JSON fallback decode path, breaks the signature-bound digest.
pub fn canonical_metadata_reservation_request_body(
request: &proto_gen::node_service::MetadataReservationRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
body.push_str(&request.disk)?;
body.push_str(&request.bucket)?;
body.push_str(&request.object)?;
body.push_u64(request.timeout_ms);
body.push_bytes(&request.bucket_incarnation_id)?;
Ok(body.finish())
}
pub fn canonical_metadata_reservation_cancel_request_body(
request: &proto_gen::node_service::MetadataReservationCancelRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.token)?;
Ok(body.finish())
}
pub fn canonical_metadata_reservation_activate_request_body(
request: &proto_gen::node_service::MetadataReservationActivateRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
body.push_str(&request.disk)?;
body.push_str(&request.bucket)?;
body.push_str(&request.object)?;
body.push_bytes(&request.token)?;
// Keep ordinary activations byte-compatible with existing peers. Renewal
// adds a distinct signed marker so an older peer cannot treat it as activation.
if request.renew_only {
body.push_bool(true);
}
Ok(body.finish())
}
pub fn canonical_reserved_rename_data_request_body(
request: &proto_gen::node_service::ReservedRenameDataRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
if let Some(rename) = request.rename.as_ref() {
body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
} else {
body.push_bytes(&[])?;
}
body.push_bytes(&request.token)?;
Ok(body.finish())
}
pub fn canonical_rename_data_request_body(
request: &proto_gen::node_service::RenameDataRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -1722,6 +1772,96 @@ mod disk_mutation_canonical_tests {
}
}
#[test]
fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
use proto_gen::node_service::{
MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
ReservedRenameDataRequest,
};
let request = MetadataReservationRequest {
disk: "disk".into(),
bucket: "bucket".into(),
object: "object".into(),
timeout_ms: 5000,
bucket_incarnation_id: vec![1; 16].into(),
};
let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
for mutate in [
|r: &mut MetadataReservationRequest| r.disk = "other".into(),
|r: &mut MetadataReservationRequest| r.bucket = "other".into(),
|r: &mut MetadataReservationRequest| r.object = "other".into(),
|r: &mut MetadataReservationRequest| r.timeout_ms = 1,
|r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
] {
let mut changed = request.clone();
mutate(&mut changed);
bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
}
assert_all_distinct(&bodies);
let activation = MetadataReservationActivateRequest {
disk: "disk".into(),
bucket: "bucket".into(),
object: "object".into(),
token: vec![1; 16].into(),
renew_only: false,
};
let mut legacy_activation = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
legacy_activation.push_str(&activation.disk).expect("disk");
legacy_activation.push_str(&activation.bucket).expect("bucket");
legacy_activation.push_str(&activation.object).expect("object");
legacy_activation.push_bytes(&activation.token).expect("token");
assert_eq!(
canonical_metadata_reservation_activate_request_body(&activation).expect("ordinary activation"),
legacy_activation.finish(),
"ordinary activation must retain its previous body digest"
);
let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
for mutate in [
|r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
|r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
|r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
|r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
|r: &mut MetadataReservationActivateRequest| r.renew_only = true,
] {
let mut changed = activation.clone();
mutate(&mut changed);
bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
}
assert_all_distinct(&bodies);
let cancellation = MetadataReservationCancelRequest {
disk: "disk".into(),
token: vec![1; 16].into(),
};
let mut changed = cancellation.clone();
changed.token = vec![2; 16].into();
assert_ne!(
canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
);
let rename = ReservedRenameDataRequest {
rename: Some(RenameDataRequest {
dst_path: "object".into(),
..Default::default()
}),
token: vec![1; 16].into(),
};
let mut changed_token = rename.clone();
changed_token.token = vec![2; 16].into();
let mut changed_object = rename.clone();
changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
assert_all_distinct(&[
canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
rename: None,
token: rename.token,
})
.expect("missing rename"),
]);
}
#[test]
fn rename_data_canonical_body_binds_every_field() {
let baseline = RenameDataRequest {
+31
View File
@@ -339,6 +339,32 @@ message RenameDataResponse {
bytes rename_data_resp_bin = 4;
}
message MetadataReservationRequest {
string disk = 1;
string bucket = 2;
string object = 3;
uint64 timeout_ms = 4;
bytes bucket_incarnation_id = 5;
}
message MetadataReservationCancelRequest {
string disk = 1;
bytes token = 2;
}
message MetadataReservationActivateRequest {
string disk = 1;
string bucket = 2;
string object = 3;
bytes token = 4;
bool renew_only = 5;
}
message ReservedRenameDataRequest {
RenameDataRequest rename = 1;
bytes token = 2;
}
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
// accepts an arbitrary volume or object path.
message PoolMetaNativeRecoveryRequest {
@@ -1314,6 +1340,11 @@ service NodeService {
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
rpc ReserveObjectMetadataShort(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
// The method itself requires source-preserving multipart publication on every peer.
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
+5 -47
View File
@@ -640,12 +640,12 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
GetRange::Offset(start) => HTTPRangeSpec {
is_suffix_length: false,
start: i64::try_from(*start).unwrap_or(i64::MAX),
start: *start as i64,
end: -1,
},
GetRange::Suffix(length) => HTTPRangeSpec {
is_suffix_length: true,
start: i64::try_from(*length).unwrap_or(i64::MAX),
start: *length as i64,
end: -1,
},
}
@@ -654,15 +654,15 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
HTTPRangeSpec {
is_suffix_length: false,
start: i64::try_from(range.start).unwrap_or(i64::MAX),
end: i64::try_from(range.end.saturating_sub(1)).unwrap_or(i64::MAX),
start: range.start as i64,
end: range.end.saturating_sub(1) as i64,
}
}
fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
HTTPRangeSpec {
is_suffix_length: false,
start: i64::try_from(start).unwrap_or(i64::MAX),
start: start as i64,
end: -1,
}
}
@@ -2476,48 +2476,6 @@ mod test {
assert_eq!(range.end, 19);
}
#[test]
fn test_get_range_conversion_for_suffix_ranges() {
// Normal suffix: last 100 bytes.
let range = http_range_spec_from_get_range(&GetRange::Suffix(100));
assert!(range.is_suffix_length);
assert_eq!(range.start, 100);
assert_eq!(range.end, -1);
// Overflowing suffix: `u64::MAX` previously wrapped to `-1` via `as i64`
// and was misread as "last 1 byte". It now clamps to `i64::MAX`, which
// `get_length` treats as "the whole object".
let range = http_range_spec_from_get_range(&GetRange::Suffix(u64::MAX));
assert!(range.is_suffix_length);
assert_eq!(range.start, i64::MAX);
assert_eq!(range.end, -1);
}
#[test]
fn test_get_range_conversion_for_offset_overflow() {
// Normal offset.
let range = http_range_spec_from_get_range(&GetRange::Offset(42));
assert!(!range.is_suffix_length);
assert_eq!(range.start, 42);
assert_eq!(range.end, -1);
// Overflowing offset clamps to `i64::MAX`, which is then rejected as
// out-of-range (416) by `get_offset_length`.
let range = http_range_spec_from_get_range(&GetRange::Offset(u64::MAX));
assert!(!range.is_suffix_length);
assert_eq!(range.start, i64::MAX);
assert_eq!(range.end, -1);
}
#[test]
fn test_get_range_conversion_for_bounded_overflow() {
// Overflowing bounded range clamps start/end to `i64::MAX`.
let range = http_range_spec_from_get_range(&GetRange::Bounded(u64::MAX - 10..u64::MAX));
assert!(!range.is_suffix_length);
assert_eq!(range.start, i64::MAX);
assert_eq!(range.end, i64::MAX);
}
#[tokio::test]
async fn test_scan_range_output_can_convert_field_delimiter() {
let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);
+2 -1
View File
@@ -27,9 +27,10 @@ pub(crate) mod fixture {
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::object::{PutObjReader, SelectObjectSnapshot};
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
#[cfg(feature = "put-object-commit-barrier")]
pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks, init_local_disks_with_instance_ctx};
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO};
#[cfg(test)]
+26 -10
View File
@@ -35,8 +35,9 @@ use std::sync::{Arc, Once};
#[cfg(feature = "put-object-commit-barrier")]
use ecstore_test_compat::fixture::ecstore_set_disk;
use ecstore_test_compat::fixture::{
BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, MakeBucketOptions, ObjectIO as _,
PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks,
BucketOperations as _, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, MakeBucketOptions,
ObjectIO as _, PoolEndpoints, PutObjReader, SelectObjectSnapshot, init_bucket_metadata_sys, init_local_disks,
init_local_disks_with_instance_ctx,
};
use tokio_util::sync::CancellationToken;
@@ -90,9 +91,8 @@ pub struct TestECStoreEnv {
pub temp_root: PathBuf,
/// The per-disk directories (`disk1`..`diskN`) under `temp_root`.
pub disk_paths: Vec<PathBuf>,
/// The store, bootstrapped exactly like the historical test setups:
/// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps
/// nextest's process-per-test parallelism safe).
/// The store, using the historical bootstrap context by default or the
/// explicit instance supplied to the builder, on `127.0.0.1:0`.
pub ecstore: Arc<ECStore>,
/// The single-pool, single-set topology the store was built from.
///
@@ -155,6 +155,7 @@ pub struct TestECStoreEnvBuilder {
prefix: String,
base_dir: Option<PathBuf>,
init_bucket_metadata: bool,
instance_context: Option<Arc<InstanceContext>>,
}
impl Default for TestECStoreEnvBuilder {
@@ -164,6 +165,7 @@ impl Default for TestECStoreEnvBuilder {
prefix: "rustfs_test_utils".to_string(),
base_dir: None,
init_bucket_metadata: true,
instance_context: None,
}
}
}
@@ -196,6 +198,12 @@ impl TestECStoreEnvBuilder {
self
}
/// Use an explicit instance instead of the process bootstrap context.
pub fn instance_context(mut self, context: Arc<InstanceContext>) -> Self {
self.instance_context = Some(context);
self
}
/// Build the environment. Panics on any bootstrap failure — this is test
/// scaffolding, and a broken environment must fail the test loudly.
pub async fn build(self) -> TestECStoreEnv {
@@ -237,14 +245,22 @@ impl TestECStoreEnvBuilder {
};
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
// Port 0 keeps ECStore-backed integration binaries parallel-safe under
// nextest: no fixed peer port is ever shared between test processes.
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr");
let ecstore = ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
.await
.expect("build test ECStore");
let ecstore = if let Some(context) = self.instance_context {
init_local_disks_with_instance_ctx(&context, endpoint_pools.clone())
.await
.expect("init isolated local disks");
ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), CancellationToken::new(), context)
.await
.expect("build isolated test ECStore")
} else {
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
.await
.expect("build test ECStore")
};
// The production bootstrap only persists pool.bin from the elected
// first cluster node. Test stores intentionally have no cluster
-1
View File
@@ -29,7 +29,6 @@ workspace = true
[features]
default = []
fips = ["rustls/fips"]
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-common/hotpath", "rustfs-config/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-common/hotpath-alloc", "rustfs-config/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-common/hotpath-cpu", "rustfs-config/hotpath-cpu"]
+5 -5
View File
@@ -1,6 +1,6 @@
[package]
name = "zstd"
version = "0.13.4"
version = "0.13.3"
publish = false
edition.workspace = true
license.workspace = true
@@ -33,10 +33,10 @@ zstd-safe = { version = "8.2.0", features = ["seekable"] }
toml = "1.1.8"
semver = "1.0.28"
apache-avro.workspace = true
arrow-array = "60.1.0"
arrow-ipc = { version = "60.1.0", features = ["zstd"] }
arrow-schema = "60.1.0"
parquet = { version = "60.1.0", default-features = false, features = ["arrow", "zstd"] }
arrow-array = "59.3.2"
arrow-ipc = { version = "59.3.2", features = ["zstd"] }
arrow-schema = "59.3.2"
parquet = { version = "59.3.2", default-features = false, features = ["arrow", "zstd"] }
kafka-protocol = { version = "0.18.0", default-features = false, features = ["zstd"] }
bytes.workspace = true
+1 -1
View File
@@ -6,6 +6,6 @@ This private package satisfies the version constraints of Avro, Arrow IPC, Parqu
The fixed implementation closes RUSTSEC-2026-0391, RUSTSEC-2026-0398, and RUSTSEC-2026-0403. A successful advisory scan alone is insufficient evidence: cargo-deny skips local path packages. The dependency-graph test therefore requires this adapter's sole runtime dependency to be the exact fixed registry implementation and rejects vulnerable registry codec versions. Security tests exercise abandoned stream contexts, malformed legacy frames, and seekable offsets past EOF. Consumer tests retain zstandard support in all four upstream formats.
Run `cargo nextest run --locked -p zstd@0.13.4` for the focused tests. The seekable feature is enabled by a test-only dependency to cover the optional vulnerable API without enabling it in production.
Run `cargo nextest run --locked -p zstd@0.13.3` for the focused tests. The seekable feature is enabled by a test-only dependency to cover the optional vulnerable API without enabling it in production.
Remove this package and the root Cargo patch when all consumers accept the fixed zstd release line directly. The cleanup marker is registered in the architecture compatibility register.
+1 -1
View File
@@ -57,7 +57,7 @@ fn legacy_constraints_resolve_only_to_the_fixed_registry_implementation() {
let version = Version::parse(package["version"].as_str().expect("package version")).expect("valid package semver");
if name == "zstd" && package.get("source").is_none() {
adapters += 1;
assert_eq!(version, Version::new(0, 13, 4));
assert_eq!(version, Version::new(0, 13, 3));
let runtime = package["dependencies"].as_array().expect("adapter dependency edges");
assert!(runtime.iter().any(|value| value.as_str() == Some("zstd 0.14.3")));
} else {
+4 -4
View File
@@ -36,6 +36,10 @@ allow-git = [
# Official s3s repository pinned for Iceberg SigV4 path encoding support.
# owner: GatewayJ review: 2026-10
"https://github.com/s3s-project/s3s",
# Upstream OIDC/OAuth2 crypto migrations, pinned to reviewed PR commits.
# owner: rustfs-maintainers review: 2026-10
"https://github.com/ramosbugs/openidconnect-rs",
"https://github.com/ramosbugs/oauth2-rs",
# Completion ownership patch fixed to rustfs/uring#20.
# owner: overtrue review: 2026-10
"https://github.com/rustfs/uring.git",
@@ -92,8 +96,4 @@ exceptions = [
# transitively via `bzip2`. Not on a hot path.
# owner: rustfs-maintainers review: 2026-07
{ allow = ["bzip2-1.0.6"], crate = "libbz2-rs-sys" },
# `aws-lc-fips-sys` declares ISC, Apache-2.0, and the OpenSSL license for
# the FIPS-validated AWS-LC module when the optional `fips` feature is enabled.
# owner: rustfs-maintainers review: 2026-10
{ allow = ["OpenSSL"], crate = "aws-lc-fips-sys" },
]
-1
View File
@@ -46,7 +46,6 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller |
| [background-services-inventory.md](background-services-inventory.md) | you need one audited background service's desired source, current-status inputs, status surface, and declared side effects |
| [scanner-usage-publication.md](scanner-usage-publication.md) | changing scanner data-usage cache publication, quota-visible usage snapshots, scanner cycle recovery, or the persisted scanner usage artifacts |
| [scanner-prefix-statistics-contract.md](scanner-prefix-statistics-contract.md) | designing, implementing, or reviewing scanner-derived prefix statistics for the object browser or another authorized consumer (proposed; not runtime implemented) |
| [scanner-dirty-journal-source-contract.md](scanner-dirty-journal-source-contract.md) | designing storage-verified provenance, completion receipts, repair responsibility or replay for the node-owned scanner dirty-producer journal (proposed; not runtime implemented) |
| [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
| [scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md) | changing what makes a scanner cycle `Complete`, `Superseded`, or converged, adding or re-classifying a dirty-usage producer, deciding whether a background writeback may invalidate a cycle, changing scanner worker concurrency or throttling, or activating scoped dirty acknowledgements |
@@ -11,8 +11,6 @@
## Open Items
- `openidconnect-hmac-signature-width` ID token verification guard: openidconnect 4.1.0 converts decoded HMAC signatures to fixed-size arrays, which panic on unexpected lengths. Check HS256, HS384, and HS512 signature widths before delegating all cryptographic, issuer, audience, expiry, and nonce checks to the upstream verifier. Remove the guard after a supported upstream release returns verification errors for arbitrary HMAC signature lengths.
- `zstd-013-fixed-backend` dependency constraint bridge: Avro, Arrow IPC, Parquet, and Kafka still require zstd 0.13. The local adapter re-exports the fixed registry zstd 0.14.3 implementation and its zstd-safe types. Remove the adapter and Cargo patch after every dependency accepts the fixed zstd line without the 0.13 bridge.
- `8565` historical rebalance replacement recovery: older schema-seven intents can exhaust their retry budget before format repair when retained rebalance metadata describes removed pools. After validating blank targets, terminal historical topology, and ownership, grant one durable handoff and one rebuild attempt. Remove after supported direct upgrades no longer admit snapshots lacking replacement_preformat_handoff_retry_done and all retained affected intents have been resolved.
@@ -64,7 +64,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. |
| Tables | Supported | Direct and staged create, register, list, load, exists, rename, commit, metadata-location get/update, and drop on both prefixes. Staged create is stateless until an `assert-create` commit atomically publishes the initial metadata and table entry. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
| Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. |
| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. Object-backed recovery reports a staged request as SUPERSEDED only when a reachable finalized same-base winner proves it lost CAS and its idempotency binding is intact. The original staged receipt is retained; its old request still conflicts. The winning history must be retained with the losing receipt; missing or contradictory history requires manual review. |
| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. |
| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
| Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. |
| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; responses carry `Cache-Control: no-store, private`. |
@@ -1,84 +0,0 @@
# Scanner Prefix Statistics Contract
**Use this when:** designing, implementing, or reviewing scanner-derived prefix statistics for the object browser or another authorized consumer.
**Status:** proposed contract; scanner-derived current-object prefix statistics, their indexed persistence, and their query API are not implemented by this document.
**Source of truth:** `crates/scanner/src/prefix_usage.rs`, `crates/data-usage/src/data_usage.rs`, `crates/scanner/src/scanner_io/io_disk.rs`, `crates/filemeta/src/metacache.rs`, `crates/ecstore/src/object_api/types.rs`, `crates/ecstore/src/store/list_objects.rs`, and `rustfs/src/storage/s3_api/bucket.rs`.
## Purpose and current boundary
A browser needs the total size of current visible objects below a displayed folder prefix and the latest modification time among those objects, including when a prefix contains very many objects. The intended result is an asynchronous scanner observation with explicit freshness and coverage, not an atomic S3 snapshot or an unconditional near-real-time guarantee. A consumer must be able to distinguish a completed observation from an incomplete, stale, or unavailable result. The requested operating scale is hundreds of millions of objects; a freshness target at that scale requires workload-specific measurement.
The existing `bucket_prefix_usage` reads per-set usage caches and exposes subtree counters and cache update time. Its `PrefixUsageSummary` does not contain current-object modification time, and scanner size accounting can include historical versions. Cache path normalization and subtree compaction can lose the literal prefix identity or the child breakdown. Consequently that interface must not be presented as this contract's current-object statistics without a separate implementation and proof.
The current S3 list path filters current delete markers and version-purge entries, and excludes free versions when selecting the visible metadata. Its response uses `ObjectInfo::get_actual_size_or_physical` and `mod_time`; for legacy data, a physical-size fallback can therefore appear where logical size is unknown. The scanner reads local metadata and has its own failure and completeness rules, so equivalent single-object decoding alone does not prove equivalence to a quorum-based listing.
This contract does not change the existing scanner usage publication, quota, repair, or S3 API guarantees. Those guarantees remain owned by [scanner usage publication](scanner-usage-publication.md), [scanner convergence](scanner-per-bucket-convergence.md), and [placement and repair invariants](placement-repair-invariants.md).
## Observable semantics
A result describes the literal requested prefix and the current visible objects covered by that prefix. The modification time is the maximum modification time of those objects, not a property supplied by S3 for the prefix itself and not the time at which a scanner cache was written. Empty coverage has a zero byte count and no object modification time only when the entire requested scope was successfully observed and is known to be empty.
Current-version selection and returned object size must be checked against the ordinary recursive `ListObjectsV2` projection, including noncurrent versions, a current delete marker, tier cleanup free versions, version-purge state, legacy compressed metadata, encryption, directory-marker objects, and absent or invalid timestamps. If the implementation cannot establish a field, it must mark that field unknown or the observation incomplete. A compatibility fallback used by S3 must be identified rather than represented as a proven logical byte count.
Prefix keys are literal S3 names, not filesystem paths. Preserve separators, percent signs, Unicode, and distinct spellings without path cleaning, repeated decoding, or prefix normalization. The API must define its supported scope, including whether it accepts only the delimiter-derived folder prefixes returned by the browser or arbitrary byte prefixes. An unsupported, compacted-away, or absent index entry is unavailable unless independently certified as empty. A trailing-slash key such as `foo/` and a literal key such as `foo__XLDIR__` can map to the same physical encoding; `crates/scanner/src/scanner_folder/prefix_attribution.rs` explicitly treats this suffix as ambiguous. A physical name or suffix decoder alone is not trusted evidence of the original key: require a verified identity source that distinguishes the represented logical key, or report the affected scope as partial or unavailable without assigning the observation to either spelling.
Counts and byte totals must use checked arithmetic. All byte totals must cross the API boundary as decimal strings, including small values, with their encoding specified by the API. Missing values, arithmetic overflow, corrupt metadata, and negative or indeterminate sizes must never become a plausible zero or saturated total.
## Observation lifecycle
The following states describe the externally visible observation, not new runtime types or implemented endpoints. Coverage and freshness remain separate fields even when the state is summarized for a consumer.
| State | Admission and transition |
|---|---|
| Unavailable | No verified compatible observation covers the request, or the caller cannot be given one; never return a fabricated zero. |
| Building | A bounded scan or rebuild is in progress; a separately verified last-known-good observation may remain available with its original coverage and age. |
| Partial | A verified candidate covers only named ranges or sets, or contains unknown fields; missing, failed, and unsupported portions remain explicit. |
| Complete | All required sets and requested ranges have compatible verified coverage, required fields, and quorum or equivalent authority for visible-version selection; this certifies an observation over a time interval, not an atomic snapshot. |
| Stale | A retained verified observation is no longer within the declared freshness target or its scope was invalidated; retain its previous coverage and timestamps. |
| Failed | A build, update, or verification failed; expose its reason without replacing a verified observation with an unverified candidate. |
A successful build can move from building to complete only after the coverage and publication checks below succeed; otherwise it remains partial or unavailable. A failed build moves to failed while a previous observation, if any, remains separately labelled stale or retains the freshness supported by its evidence. Retry starts another bounded build; cancellation stops that build without changing the last verified observation. A bucket incarnation or layout mismatch invalidates the affected identity rather than relabelling its data for a new bucket or set.
## Coverage and freshness
A result carries its coverage and freshness independently. Internally, its evidence binds the bucket incarnation and requested literal prefix, contributing and expected set identities, each set's observation interval or oldest and newest represented observations, the completeness of its represented scope, and reasons for missing or unknown fields. The response exposes only the authorized, redacted coverage and freshness needed to interpret the result, as required below. A consumer may show a partial observed total only if it is labelled partial and its covered scope is explicit.
A completed traversal proves only the coverage actually certified by that traversal; it does not create a point-in-time snapshot across sets or concurrent writes. A last-known-good shard or reused subtree retains its own oldest observation time. A publication timestamp, latest set update, or cache invalidation time must not be substituted for the time range of the data represented. Traversal completion on one local disk is not proof of an authoritative visible version. Before a scope is complete, require the same cross-disk quorum and version-selection evidence used by S3 listing, or separately verified equivalent evidence; stale replicas, competing delete markers, unavailable disks, and conflicting cross-pool migration observations must remain partial or unavailable until resolved. For each object, replica disagreement or missing authority cannot be settled by choosing whichever metadata the scanner saw first.
Before any valid observation exists, the result is unavailable or building. A bounded traversal may publish an explicitly partial result when its covered ranges and unresolved gaps are known; after all required scope is certified it may publish a complete observation. A later mutation, expired freshness target, incomplete retry, topology change, unreadable set, or failed verification marks the affected result stale, partial, or unavailable as appropriate. A reader must never infer currentness solely from an earlier complete traversal.
The result must expose the distinction between coverage and age even when a consumer requests a single summary. A complete but old result remains complete for its recorded observation and stale for current use; a recently updated partial result remains partial. The deployment must define freshness targets and report measured observation lag rather than claim an unverified deadline.
Freshness certification requires evidence that relevant mutations and generation changes in the represented scope are covered through the claimed observation boundary. A dirty notification or local generation alone is not proof that every producer, peer, pending physical tail, or data-movement path was observed. A missing producer, unknown generation, replay gap, mixed-version peer, lost notification, or unverified journal interval must invalidate the freshness claim and leave the affected observation stale, partial, or unavailable until coverage is re-established. This document does not assert that a complete mutation journal or such certification already exists.
## Persistence and recovery
The index requires a separately versioned persistence contract. Reuse the scanner's admission, pacing, traversal, checkpoint, and coverage mechanisms only where their identities and guarantees match the indexed scope. Do not change the meaning of the existing `.usage-cache.bin` data or quota-visible totals to store the new result.
Partition persistent index data so that reading or updating one scope does not require loading all prefixes in a bucket into memory. A manifest identifies its immutable bounded shards, literal key ranges, format version, bucket incarnation, topology and set identities, scanner generation, observation intervals, coverage, and integrity evidence. The supported prefix query and shard bounds must be explicit; pagination and continuation tokens must bind the requested scope and compatible generation. Validate untrusted cursors for authenticity, expiry, version, bounds, principal and authorized scope, bucket incarnation, prefix, and generation before use; reject altered, cross-scope, stale, or unsupported cursors without leaking indexed data.
Publish a generation only after all referenced shard writes are durable and verified and the active writer's leadership, bucket incarnation, topology, and publication fences remain valid at the commit point. A concrete implementation must establish the atomic publication or conditional replacement primitive and its failure behavior; this document does not assert that a suitable primitive already exists. Readers validate the manifest and shard identity before use and retain a previous verified generation only with its original freshness and coverage labels.
Interrupted work may resume only from a validated checkpoint bound to the same scope, generation, bucket incarnation, layout, and coverage evidence. Otherwise discard the candidate or rebuild it; do not combine incompatible generations or count both a replayed segment and its prior contribution. The release design must handle competing writers, crashes before and after publication, missing shards, failed reads, corruption, leader replacement, storage exhaustion, bucket deletion and recreation, resharding, rollback, and orphan cleanup.
Unknown format versions and legacy records lacking the new proof are unavailable, not empty. During rolling upgrades, an older writer must not erase a newer index or certify new fields as complete; keep independent index ownership and version fencing, and provide a supported rollback path. A stopped or failed index must not block object writes, lifecycle, healing, or existing scanner usage publication.
## Authorization and disclosure
An index query must authenticate and authorize the caller for the actual bucket and literal prefix before reading or returning index data. Its decision must respect the same applicable `ListBucket` policy semantics as the corresponding recursive S3 listing, including prefix conditions, bucket policy, explicit denies, session restrictions, and current credentials. The operation must not use a broader background or administrator identity to disclose information unavailable to the caller.
A parent-prefix aggregate may reveal objects in a restricted descendant. The implementation must establish that the caller can obtain the represented scope through the corresponding listing policy or return no aggregate for that scope. Pagination, cached responses, progress, error details, and last-known-good data are subject to the same authorization and must not expose other buckets, prefixes, or principals. Reauthorize each request; a cached allow decision must not survive a relevant policy or credential change without a proven invalidation mechanism. Ordinary users receive only the coverage and freshness information needed to interpret their authorized result; set identifiers, internal topology, physical paths, and detailed failure provenance require a separately authorized administrative surface. Redaction must also apply to cursors, progress, and error messages.
The existing admin prefix-usage handler uses the shared admin gate before parsing the requested bucket. Its current authorization and response semantics must not be assumed to implement this contract; an integration must establish and test the actual resource and query-condition decision before using that route.
## Resource and lifecycle boundaries
Index construction and refresh are background work. Bound admission, queue depth, per-bucket and per-tenant concurrency, scan work per scheduling quantum, disk and network I/O, memory, shard size, retained generations, and disk usage. Cancellation and restart must stop or resume work without exposing unfinished data as complete. Refresh, compaction, cleanup, and a slow or unavailable set must not starve foreground object operations.
A design must avoid accumulating the object population or an unbounded set of prefix rows in process memory, or issuing a full object-list scan for every displayed row. Query admission must enforce independent hard limits on requested page size, shards visited, bytes read, CPU time, memory, concurrent requests, and request duration; when a budget is exhausted, return a bounded partial or unavailable result with an appropriate continuation only when continuation is safe. It must account for key depth and write amplification, large single prefixes, many sibling prefixes, pagination, topology changes, and backpressure. Resource exhaustion must produce a visible degraded or unavailable state and retain object-store correctness; it must not cause silent data loss or an unbounded retry loop.
## Required evidence
Before the indexed result is enabled, tests must compare its current-object projection with S3 listings for ordinary, versioned, deleted, transitioned, legacy, corrupt, and unusual literal keys. Integration tests must cover scope-limited allow and deny policies, session changes, pagination and hostile cursors, internal-topology redaction, multiple sets, incomplete or corrupt shards, old formats, mixed versions, deletion and recreation, leader changes, cancellation, retries, and concurrent object mutations. Differential and fault tests must cover `foo/` versus literal `foo__XLDIR__`, stale replicas, competing delete markers, unavailable quorum, cross-pool migration conflicts, and gaps in mutation or generation evidence; no such case may silently certify a guessed key or version.
Recovery tests must inject failure around shard writes, manifest publication, checkpoint replay, and cleanup and prove that readers never certify missing or mismatched data. Performance validation must measure scan completion and observation lag, memory and persistent-space growth, I/O load, and foreground workload impact under representative large populations and concurrent writes. Claims for the requested hundreds-of-millions scale and any near-real-time target require measured evidence at the stated scale and workload; a unit test, small successful scan, or unsupported extrapolation does not establish them. Larger scale claims require their own target and evidence and are not implied by this contract.
@@ -1,29 +1,29 @@
# Cryptographic compliance positioning
**Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs` / `rustfs-tls-runtime` / `rustfs-crypto` `fips` features and algorithm deprecation.
**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `rustfs/Cargo.toml` and `crates/tls-runtime/Cargo.toml` (`fips` feature); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`).
**Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs-crypto` `fips` feature and algorithm deprecation.
**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`).
This document records where RustFS stands on cryptographic module validation and what may and may not be said about it, so the question is answered once, from the code. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md).
## Status: not FIPS 140-3 validated
**RustFS is not FIPS 140-3 (or 140-2) validated as a product, and in the default build no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS itself, or the libraries it uses in the default shipped configuration. An opt-in `--features fips` build flag on `rustfs` switches the `rustls` / `aws-lc-rs` provider to `aws-lc-fips-sys` for TLS, X.509, and JWT operations, while object data-path and local KMS operations remain on unvalidated RustCrypto software implementations. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS as a whole has or pursues.
**RustFS is not FIPS 140-3 (or 140-2) validated, and no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS or the libraries it uses in the shipped configuration. This is a deliberate position and not a statement about algorithm strength: the algorithms in use are standard, well-reviewed AEADs. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS has or pursues.
### What the process actually links
| Layer | Where | Implementation | Primitives | Validated module |
| --- | --- | --- | --- | --- |
| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider (`default_provider` by default; `default_fips_provider` when built with `--features fips`) | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange (restricted to FIPS-approved suites and groups when `--features fips` is enabled) | No in the default build (`aws-lc-sys`); opt-in `--features fips` links `aws-lc-fips-sys` and asserts `provider.fips()` at startup |
| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange | No — the ordinary `aws-lc-rs` build, not the `aws-lc-fips-sys`-backed FIPS variant |
| Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No |
| DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No |
| Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No |
| Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `rustfs-crypto` `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS `aws-lc-sys` build by default; unified with `aws-lc-fips-sys` when `rustfs` is built with `--features fips` | No in the default build; links `aws-lc-fips-sys` only when built with `-p rustfs --features fips` |
| Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS build | No |
Two consequences are commonly assumed the other way:
- **AWS-LC being present does not imply FIPS in the default build.** `aws-lc-rs` has a FIPS variant (`aws-lc-fips-sys`); default builds do not enable it. Building `rustfs` with `--features fips` enables `rustls/fips`, `tokio-rustls/fips`, and `rustfs-tls-runtime/fips`, which links `aws-lc-fips-sys` and installs `rustls::crypto::default_fips_provider()` (see [below](#the-rustfs-and-rustfs-tls-runtime-fips-feature)).
- **The object data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations, even when `rustfs` is built with `--features fips`. Enabling the TLS FIPS provider does not change the object encryption data path; see route 1 below.
- **AWS-LC being present does not imply FIPS.** `aws-lc-rs` has a FIPS variant; the workspace does not enable it anywhere.
- **The data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations. Swapping the TLS provider would not change that; see route 1 below.
## Terminology red lines for external material
@@ -50,15 +50,6 @@ Boilerplate when the topic cannot be avoided:
`scripts/check_fips_wording.sh` greps `README.md` and `CHANGELOG.md` for the banned phrases above, and separately rejects any wording under `crates/kms` that describes the Vault KV2 backend as wrapping key material through Vault's Transit engine (`KmsBackend::VaultKv2` stores RustFS-wrapped material in KV v2 and never calls Transit; use `VaultTransit` when cryptographic isolation is the requirement). This document is intentionally outside the scan: it needs the terminology to define the policy.
## The `rustfs` and `rustfs-tls-runtime` `fips` feature
`rustfs/Cargo.toml` (`fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]`) and `crates/tls-runtime/Cargo.toml` (`fips = ["rustls/fips"]`) define an opt-in Cargo feature (disabled by default) for builds that require the `aws-lc-fips-sys` cryptographic module on the `aws-lc-rs` / `rustls` path:
- **What it enables:** Activates `rustls/fips` and `tokio-rustls/fips`, which pull in `aws-lc-rs/fips` (`aws-lc-fips-sys`) and `rustls-webpki/aws-lc-rs-fips`. Because Cargo unifies features across the dependency graph, other `aws-lc-rs` consumers in the binary (`jsonwebtoken`, `rcgen`, `google-cloud-auth`, `async-nats`) also link against `aws-lc-fips-sys`.
- **Runtime provider installation:** In `rustfs/src/startup_runtime_hooks.rs`, `build_default_crypto_provider()` selects `rustls::crypto::default_fips_provider()` (instead of `rustls::crypto::aws_lc_rs::default_provider()`) and asserts `provider.fips()` on startup, restricting TLS negotiation to FIPS-approved cipher suites (e.g., `TLS_AES_256_GCM_SHA384`, `TLS_AES_128_GCM_SHA256`, `TLS_ECDHE_*_AES_*_GCM_*`) and key exchange groups (excluding `TLS_CHACHA20_POLY1305_SHA256` and `X25519`).
- **Build requirements:** Compiling `aws-lc-fips-sys` requires CMake, Go, and Clang/`libclang` in the build environment (unlike the default `aws-lc-sys` build).
- **What it does not change:** It does not alter the object data-path AEAD (`crates/kms`, `crates/rio`, `crates/rio-v2`) or `rustfs-crypto` config/IAM blob encryption, which continue to use pure-Rust `RustCrypto` crates.
## The `rustfs-crypto` `fips` feature: what it actually does
`crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way.
@@ -80,7 +71,7 @@ Selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src
| Route | Scope | What it buys | Verdict |
| --- | --- | --- | --- |
| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) across the data path | The TLS/X.509/JWT provider swap is wired via the opt-in `-p rustfs --features fips` build flag. Extending `aws-lc-fips-sys` to the data path would require re-implementing every AEAD/KDF call in `crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs CMake, a C toolchain, Clang/`libclang`, and Go, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | The accurate claim becomes "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated"; a product-level claim additionally needs a documented boundary, approved-mode enforcement, self-tests, zeroization, and entropy documentation | Data-path rewrite remains heavy and re-opens platform-support constraints; TLS/X.509/JWT is opt-in via `--features fips` |
| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) | The TLS provider swap is the small part. Every AEAD call in the data path (`crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, `crates/crypto/src/encdec/`) would be re-implemented against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs CMake, a C toolchain, and Go, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | The accurate claim becomes "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated"; a product-level claim additionally needs a documented boundary, approved-mode enforcement, self-tests, zeroization, and entropy documentation | Heavy; re-opens an open platform-support problem. Justified only by a named customer or regulatory commitment |
| 2. Let an externally validated KMS carry key operations | Mostly built: the `VaultTransit` backend never lets key-encryption key material leave Vault, so master key generation, wrapping, unwrapping, and rotation happen inside whatever module Vault's seal/HSM is validated against. Remaining work is configuration guidance and a supported-deployment statement | A defensible partial answer to "where do keys live and who validated that". The object data path stays RustCrypto and TLS stays non-FIPS AWS-LC: "key management operations are performed by an externally validated module; the object data path is not validated" | Nearest partial step, no code rewrite. Point customers here when the requirement is key custody rather than a certificate covering the storage layer |
| 3. Make no validation claim (current default) | This document plus the grep guard | Nothing further | The current decision. FIPS 140-3 validation is not a roadmap target; adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI |
+2 -2
View File
@@ -14,7 +14,7 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
| Outbound target matrix | Replication of every object shape (empty, plain, retention, legal hold, multipart, checksummed, SSE-S3) against every remote-target failure mode the fake target models; an explicit expectation table pins known-red cells to an open issue | `python3 scripts/e2e_binary.py build`, then `python3 scripts/e2e_binary.py run -- cargo nextest run -p e2e_test -E 'test(/^replication_target_matrix_test::/)'` | With `e2e-repl-nightly`; required locally for any change to outbound client defaults (SOP: [`docs/postmortems/2026-09-03-replication-checksum-default-regression.md`](../postmortems/2026-09-03-replication-checksum-default-regression.md)) |
| s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | Non-documentation PRs, required through `Test and Lint` (second half of the `End-to-End Tests` job) |
| S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | Non-documentation PRs: implemented and lifecycle S3 lanes required through `Test and Lint`; the separate s3-tests workflow provides additional PR reporting and a weekly full sweep; mint: weekly, report-only |
| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos nightly via `e2e-distributed` |
| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` |
| Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
| Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |
| Conditional reads | Paired ordinary/hit/miss GET latency, throughput and read-lock measurements | `scripts/run_conditional_read_abba.py` | Local conditional-read acceptance; [guide and scope](conditional-read-benchmark.md) |
@@ -88,7 +88,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
| `e2e-full` | Merge-queue / main-push single-node e2e lane |
| `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
| `e2e-nightly` | Nightly serial multi-process cluster fault lane |
| `e2e-distributed` | Nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
| `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |
Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):
+4 -3
View File
@@ -35,7 +35,7 @@ gh api repos/rustfs/rustfs/rulesets/6436880 \
--jq '.rules[] | select(.type == "required_status_checks") | .parameters'
```
The aggregate requires the validation lanes already selected by `ci.yml`; this closes the gap where a failing critical lane left the required workspace check green. Independent workflows remain report-only unless separately required. Before adding a new expensive lane or moving existing PR coverage to a schedule, collect representative execution and regression evidence, establish ownership and a working scheduled replacement, and update this reference with the resulting policy. `e2e-distributed.yml` and `e2e-upgrade.yml` are that kind of replacement: the 4-node lane runs on its nightly schedule, and the single-node upgrade lane runs on its weekly Monday schedule, on version tags, and on manual dispatch. Neither is part of `Test and Lint`. Dispatch the workflow against a candidate branch when that signal is needed before merge.
The aggregate requires the validation lanes already selected by `ci.yml`; this closes the gap where a failing critical lane left the required workspace check green. Independent workflows remain report-only unless separately required. Before adding a new expensive lane or moving existing PR coverage to a schedule, collect representative execution and regression evidence, establish ownership and a working scheduled replacement, and update this reference with the resulting policy.
Superseded PR attempts are cancelled. A running main validation finishes, with only the newest pending main run retained, so frequent merges cannot continuously cancel the full baseline. Having a `merge_group` trigger does not itself require use of the merge queue.
@@ -66,6 +66,7 @@ The four site-replication state-writer concurrency proofs reserve the available
| PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
| PR touching `paths` in `windows-filesystem.yml` | `Rename Safety`, `Startup Smoke` | `windows-filesystem.yml` `rename-safety`, `startup-smoke` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows; `cargo build --locked -p rustfs --bin rustfs`, then `rustfs.exe --version` and a single-volume server polled on `/health/ready` |
| PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
| PR touching `paths` in `e2e-upgrade.yml` | `Upgrade compatibility` | `e2e-upgrade.yml` `upgrade` | Report-only | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
| PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
| PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
| PR, documentation-only selection | `Quick Checks`, `Typos`, `Test and Lint` | `ci.yml` `quick-checks`, `typos`, `required-checks` | Required directly or via aggregate | Quick Checks commands; `python3 scripts/ci_gate.py --self-test` |
@@ -89,7 +90,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
| `cache-warm.yml` (every six hours, plus main pushes that change `Cargo.lock`, `rust-toolchain.toml` or the setup action) | `warm-ci-dev`, `warm-ci-feat-rio`, `warm-ci-feat-proto`, `warm-ci-uring` | no verdict; sole writer of the `rust-cache` keys that `ci.yml` restores | yes | dispatch `cache-warm.yml` |
| `build.yml` (nightly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
| `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--features e2e-test-hooks` on both steps with `--profile e2e-nightly`; `--features ftps,webdav,sftp` on both steps with `-j 1 --profile e2e-protocols` |
| `e2e-distributed.yml` (nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, run `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test` |
| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, run `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test` |
| `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
| `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes (disabled) | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
| `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
@@ -99,7 +100,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
| `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
| `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
| `e2e-upgrade.yml` (weekly) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | see the PR row |
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |
+10 -9
View File
@@ -273,9 +273,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.157.0"
version = "1.156.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea9794c741520d8baec9462bcd0ad12164d4f5f78d008a8da9d6a0b7222caa8"
checksum = "1072c9aa20159b9b417115342d38c4c53108fc98e5359fcb86b7d4575747eb59"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -1837,18 +1837,18 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.30.3"
version = "0.30.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935"
checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e"
dependencies = [
"hotpath-macros",
]
[[package]]
name = "hotpath-macros"
version = "0.30.3"
version = "0.30.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a"
checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58"
[[package]]
name = "http"
@@ -2418,9 +2418,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.35"
version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
[[package]]
name = "lru"
@@ -4134,7 +4134,8 @@ dependencies = [
[[package]]
name = "rustfs-uring"
version = "0.3.0"
source = "git+https://github.com/rustfs/uring.git?rev=eb8fc160e71d804427de1e905f5c226d580f71bf#eb8fc160e71d804427de1e905f5c226d580f71bf"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7deda770b7981ee3bfaa202ecd18625fc794c962b8d58cd4f436952b7e66dcf"
dependencies = [
"io-uring",
"libc",
-1
View File
@@ -58,7 +58,6 @@ required-features = ["swift"]
[features]
default = ["ftps", "webdav", "gcs"]
fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips"]
gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
metrics-gpu = ["rustfs-obs/gpu"]
ftps = ["rustfs-protocols/ftps"]
+3 -22
View File
@@ -14,10 +14,7 @@
use crate::license::license_status;
use crate::startup_runtime_sources;
#[cfg(not(feature = "fips"))]
use rustls::crypto::aws_lc_rs::default_provider;
#[cfg(feature = "fips")]
use rustls::crypto::default_fips_provider as default_provider;
use std::future::Future;
use std::io::Result;
use tracing::{debug, info, warn};
@@ -108,18 +105,8 @@ where
shutdown();
}
fn build_default_crypto_provider() -> rustls::crypto::CryptoProvider {
let provider = default_provider();
#[cfg(feature = "fips")]
assert!(
provider.fips(),
"FIPS crypto provider requested via `fips` feature, but AWS-LC FIPS mode is not active"
);
provider
}
pub(crate) fn install_default_crypto_provider() {
if build_default_crypto_provider().install_default().is_err() {
if default_provider().install_default().is_err() {
debug!(
target: "rustfs::main",
event = EVENT_CRYPTO_PROVIDER_STATE,
@@ -168,7 +155,7 @@ pub(crate) async fn init_embedded_runtime_hooks(obs_endpoint: String) -> Result<
}
fn install_embedded_default_crypto_provider() {
if let Err(err) = build_default_crypto_provider().install_default() {
if let Err(err) = default_provider().install_default() {
debug!(
component = LOG_COMPONENT_EMBEDDED,
subsystem = LOG_SUBSYSTEM_EMBEDDED,
@@ -182,18 +169,12 @@ fn install_embedded_default_crypto_provider() {
#[cfg(test)]
mod tests {
use super::{build_default_crypto_provider, init_profiling_runtime_with, shutdown_profiling_runtime_with};
use super::{init_profiling_runtime_with, shutdown_profiling_runtime_with};
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
#[test]
fn default_crypto_provider_matches_fips_feature() {
let provider = build_default_crypto_provider();
assert_eq!(provider.fips(), cfg!(feature = "fips"));
}
#[tokio::test]
async fn init_profiling_runtime_invokes_registered_hook() {
let called = Arc::new(AtomicBool::new(false));
+888 -23
View File
@@ -1826,6 +1826,41 @@ impl Node for NodeService {
self.handle_rename_data(request, false, true, false).await
}
async fn reserve_object_metadata(
&self,
request: Request<MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
self.handle_reserve_object_metadata(request, false).await
}
async fn reserve_object_metadata_short(
&self,
request: Request<MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
self.handle_reserve_object_metadata(request, true).await
}
async fn cancel_object_metadata_reservation(
&self,
request: Request<MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
self.handle_cancel_object_metadata_reservation(request).await
}
async fn activate_object_metadata_reservation(
&self,
request: Request<MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
self.handle_activate_object_metadata_reservation(request).await
}
async fn rename_reserved_data(
&self,
request: Request<ReservedRenameDataRequest>,
) -> Result<Response<RenameDataResponse>, Status> {
self.handle_rename_reserved_data(request).await
}
async fn undo_pool_meta_write(
&self,
request: Request<DeleteVersionRequest>,
@@ -3245,15 +3280,17 @@ mod tests {
HealControlRequest, HealControlResponse, ListBucketRequest, ListDirRequest, ListVolumesRequest,
LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest,
LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest,
MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest, MetadataReservationCancelRequest,
MetadataReservationRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest,
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
RenameFileRequest, RenamePartRequest, ReservedRenameDataRequest, ScannerActivityRequest,
ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest,
StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState,
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -3271,8 +3308,13 @@ mod tests {
use tonic::{Request, Response, Status};
use uuid::Uuid;
const DISK_MUTATION_RPC_METHODS: [&str; 34] = [
const DISK_MUTATION_RPC_METHODS: [&str; 39] = [
"renamedata",
"reserveobjectmetadata",
"reserveobjectmetadatashort",
"activateobjectmetadatareservation",
"cancelobjectmetadatareservation",
"renamereserveddata",
"renamemultipartdata",
"recovermultipartsource",
"renamedataatincarnation",
@@ -4970,6 +5012,55 @@ mod tests {
},
rustfs_protos::canonical_rename_data_request_body
);
assert_gated!(
reserve_object_metadata,
MetadataReservationRequest {
disk: disk.clone(),
bucket: "bucket".into(),
object: "object".into(),
timeout_ms: 5000,
bucket_incarnation_id: vec![1; 16].into(),
},
rustfs_protos::canonical_metadata_reservation_request_body
);
assert_gated!(
reserve_object_metadata_short,
MetadataReservationRequest {
disk: disk.clone(),
bucket: "bucket".into(),
object: "object".into(),
timeout_ms: 5000,
bucket_incarnation_id: vec![1; 16].into(),
},
rustfs_protos::canonical_metadata_reservation_request_body
);
assert_gated!(
activate_object_metadata_reservation,
MetadataReservationActivateRequest {
disk: disk.clone(),
bucket: "bucket".into(),
object: "object".into(),
token: Uuid::new_v4().as_bytes().to_vec().into(),
renew_only: false,
},
rustfs_protos::canonical_metadata_reservation_activate_request_body
);
assert_gated!(
cancel_object_metadata_reservation,
MetadataReservationCancelRequest {
disk: disk.clone(),
token: Uuid::new_v4().as_bytes().to_vec().into(),
},
rustfs_protos::canonical_metadata_reservation_cancel_request_body
);
assert_gated!(
rename_reserved_data,
ReservedRenameDataRequest {
rename: Some(RenameDataRequest::default()),
token: Uuid::new_v4().as_bytes().to_vec().into(),
},
rustfs_protos::canonical_reserved_rename_data_request_body
);
assert_gated!(
recover_multipart_source,
RecoverMultipartSourceRequest::default(),
@@ -7475,13 +7566,26 @@ mod tests {
}
async fn target_rpc_fixture() -> TargetRpcFixture {
target_rpc_fixture_with_bucket_metadata(false).await
}
async fn target_rpc_fixture_with_bucket_metadata(initialize_bucket_metadata: bool) -> TargetRpcFixture {
super::timeout(Duration::from_secs(90), async {
let root = tempfile::tempdir().expect("target RPC root");
let env = rustfs_test_utils::TestECStoreEnv::builder()
let instance = if initialize_bucket_metadata {
crate::storage::storage_api::new_instance_ctx()
} else {
crate::storage::storage_api::bootstrap_instance_ctx()
};
let builder = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(root.path())
.init_bucket_metadata(false)
.build()
.await;
.init_bucket_metadata(initialize_bucket_metadata);
let builder = if initialize_bucket_metadata {
builder.instance_context(instance.clone())
} else {
builder
};
let env = builder.build().await;
ObjectStore::new(env.ecstore.clone())
.save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
.await
@@ -7494,10 +7598,9 @@ mod tests {
iam.clone(),
Arc::new(KmsServiceManager::new()),
));
let instance = crate::storage::storage_api::bootstrap_instance_ctx();
assert!(
super::BootstrapLocalTarget::new(instance.clone()).is_for_store(&env.ecstore),
"the standard builder must use this exact instance context"
"the fixture must use this exact instance context"
);
super::timeout(Duration::from_secs(10), async {
while env.ecstore.namespace_commits_pending_for_test() {
@@ -8146,6 +8249,13 @@ mod tests {
}
async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
stage_target_rpc_inner(fixture, true).await
}
async fn stage_target_rpc_inner(
fixture: &TargetRpcFixture,
create_volume: bool,
) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions};
let set = fixture
.env
@@ -8168,7 +8278,9 @@ mod tests {
}];
fi.data = Some(Bytes::from_static(b"target-rpc-inline"));
fi.set_inline_data();
disk.make_volume("target-rpc").await.expect("target volume");
if create_volume {
disk.make_volume("target-rpc").await.expect("target volume");
}
disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone())
.await
.expect("stage real inline body");
@@ -8220,6 +8332,740 @@ mod tests {
request
}
#[tokio::test]
#[serial_test::serial]
async fn isolated_reservation_fixture_does_not_capture_gating_store() {
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("rpc-isolated-order", false).await;
let (_store, bucket) = tokio::time::timeout(
Duration::from_secs(20),
crate::app::gating_test_env::durable_quota_test_bucket("rpc-isolated-order", 1024),
)
.await
.expect("gating fixture initializes after isolated RPC fixture");
assert!(!bucket.is_empty());
}
#[tokio::test]
#[serial_test::serial]
async fn lost_short_reservation_rpc_response_releases_the_bucket_fence() {
use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("target-rpc", false).await;
let bucket = "target-rpc-short-lost";
fixture.env.make_bucket(bucket, false).await;
let incarnation = fixture
.env
.ecstore
.bucket_incarnation_id(bucket)
.await
.expect("bucket incarnation");
let (disk, _, _) = stage_target_rpc_inner(&fixture, false).await;
let lock = fixture
.env
.ecstore
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.expect("publication lock");
let service = make_server_for_context(Some(fixture.context.clone()));
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener");
let addr = listener.local_addr().expect("RPC listener address");
let (shutdown, stopped) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
tonic::transport::Server::builder()
.add_service(NodeServiceServer::new(service))
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
let _ = stopped.await;
})
.await
.expect("serve RPC target");
});
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
.await
.expect("connect RPC client");
let mut request = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body");
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
mark_v2_authenticated(&mut request);
let mut pause =
super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Reserve);
let call = tokio::spawn(async move { client.reserve_object_metadata_short(request).await });
tokio::time::timeout(Duration::from_secs(5), &mut pause.entered)
.await
.expect("server allocated the token")
.expect("server signalled allocation");
assert!(
tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5)))
.await
.is_err(),
"the token owns the bucket fence before response delivery"
);
call.abort();
let _ = call.await;
drop(pause);
let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4)))
.await
.expect("the lost response does not strand the bucket fence")
.expect("publication writer acquires");
drop(writer);
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
.await
.expect("reconnect RPC client");
let mut request = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body");
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
mark_v2_authenticated(&mut request);
let prepared = client
.reserve_object_metadata_short(request)
.await
.expect("short reservation")
.into_inner();
assert!(prepared.success, "short reservation must succeed");
let mut renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
token: prepared.token,
renew_only: true,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renewal body");
set_tonic_canonical_body_digest(&mut renew, &body).expect("renewal digest");
mark_v2_authenticated(&mut renew);
let mut pause =
super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Renew);
let call = tokio::spawn(async move { client.activate_object_metadata_reservation(renew).await });
tokio::time::timeout(Duration::from_secs(5), &mut pause.entered)
.await
.expect("server accepted renewal")
.expect("server signalled renewal");
assert!(
tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5)))
.await
.is_err(),
"the renewed token must retain the bucket fence"
);
call.abort();
let _ = call.await;
drop(pause); // Model loss of the renewal response and subsequent cancel.
let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4)))
.await
.expect("lost renewal and cancel do not strand the bucket fence")
.expect("publication writer acquires after renewal expiry");
drop(writer);
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
.await
.expect("reconnect RPC client");
let mut request = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body");
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
mark_v2_authenticated(&mut request);
let prepared = client
.reserve_object_metadata_short(request)
.await
.expect("short reservation")
.into_inner();
assert!(prepared.success, "short reservation must succeed");
let mut activate = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
token: prepared.token,
renew_only: false,
});
let body =
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activation body");
set_tonic_canonical_body_digest(&mut activate, &body).expect("activation digest");
mark_v2_authenticated(&mut activate);
let mut pause =
super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Activate);
let call = tokio::spawn(async move { client.activate_object_metadata_reservation(activate).await });
tokio::time::timeout(Duration::from_secs(5), &mut pause.entered)
.await
.expect("server activated reservation")
.expect("server signalled activation");
assert!(
tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5)))
.await
.is_err(),
"the activated token must retain the bucket fence"
);
call.abort();
let _ = call.await;
drop(pause); // Model loss of the activation response and subsequent cancel.
let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4)))
.await
.expect("lost activation and cancel do not strand the bucket fence")
.expect("publication writer acquires after activation expiry");
drop(writer);
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
.await
.expect("reconnect RPC client");
let mut request = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body");
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
mark_v2_authenticated(&mut request);
let prepared = client
.reserve_object_metadata_short(request)
.await
.expect("short reservation")
.into_inner();
assert!(prepared.success, "short reservation must succeed");
let mut activate = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
token: prepared.token.clone(),
renew_only: false,
});
let body =
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activation body");
set_tonic_canonical_body_digest(&mut activate, &body).expect("activation digest");
mark_v2_authenticated(&mut activate);
assert!(
client
.activate_object_metadata_reservation(activate)
.await
.expect("activation RPC")
.into_inner()
.success,
"short token must activate"
);
let mut renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: bucket.into(),
object: "destination".into(),
token: prepared.token,
renew_only: true,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renewal body");
set_tonic_canonical_body_digest(&mut renew, &body).expect("renewal digest");
mark_v2_authenticated(&mut renew);
let mut pause =
super::disk::short_response_test_hook::install(bucket, super::disk::short_response_test_hook::Phase::Renew);
let call = tokio::spawn(async move { client.activate_object_metadata_reservation(renew).await });
tokio::time::timeout(Duration::from_secs(5), &mut pause.entered)
.await
.expect("server accepted queued renewal")
.expect("server signalled queued renewal");
call.abort();
let _ = call.await;
drop(pause);
let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4)))
.await
.expect("lost queued renewal and cancel do not strand the bucket fence")
.expect("publication writer acquires after queued renewal expiry");
drop(writer);
let _ = shutdown.send(());
server.await.expect("server task");
}
#[tokio::test]
#[serial_test::serial]
async fn short_metadata_reservation_expires_and_preserves_legacy_lifetime() {
use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("target-rpc", false).await;
let incarnation = fixture
.env
.ecstore
.bucket_incarnation_id("target-rpc")
.await
.expect("bucket incarnation");
let (disk, _, _) = stage_target_rpc_inner(&fixture, false).await;
let service = make_server_for_context(Some(fixture.context.clone()));
let lock = fixture
.env
.ecstore
.new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.expect("publication lock");
let make_request = || {
let mut request = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref()).expect("reservation body");
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
mark_v2_authenticated(&mut request);
request
};
let writer = lock
.get_write_lock(Duration::from_secs(1))
.await
.expect("hold publication writer");
let refused = tokio::time::timeout(Duration::from_millis(500), service.reserve_object_metadata_short(make_request()))
.await
.expect("short acquisition is bounded")
.expect("short handler")
.into_inner();
assert!(!refused.success, "an occupied publication lock must not issue a token");
assert_eq!(refused.protocol_version, 3);
drop(writer);
let response = service
.reserve_object_metadata_short(make_request())
.await
.expect("short handler")
.into_inner();
assert!(response.success);
assert_eq!(response.protocol_version, 3);
drop(response); // Model a successful server response that never reaches its caller.
assert!(
tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5)))
.await
.is_err(),
"live token retains the fence"
);
let writer = tokio::time::timeout(Duration::from_secs(4), lock.get_write_lock(Duration::from_secs(4)))
.await
.expect("short token releases after expiry")
.expect("publication writer acquires");
drop(writer);
let legacy = service
.reserve_object_metadata(make_request())
.await
.expect("legacy handler")
.into_inner();
assert!(legacy.success);
assert_eq!(legacy.protocol_version, 2);
tokio::time::sleep(Duration::from_secs(3)).await;
assert!(
tokio::time::timeout(Duration::from_millis(100), lock.get_write_lock(Duration::from_secs(5)))
.await
.is_err(),
"legacy token remains live"
);
let mut cancel = Request::new(MetadataReservationCancelRequest {
disk: disk.endpoint().to_string(),
token: legacy.token,
});
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(cancel.get_ref()).expect("cancel body");
set_tonic_canonical_body_digest(&mut cancel, &body).expect("cancel digest");
mark_v2_authenticated(&mut cancel);
assert!(
service
.cancel_object_metadata_reservation(cancel)
.await
.expect("cancel handler")
.into_inner()
.success
);
lock.get_write_lock(Duration::from_secs(1))
.await
.expect("cancel releases legacy fence");
}
#[tokio::test]
#[serial_test::serial]
async fn metadata_reservation_handlers_bind_and_publish_on_the_target_disk() {
use crate::storage::storage_api::ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions};
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("target-rpc", false).await;
let incarnation = fixture
.env
.ecstore
.bucket_incarnation_id("target-rpc")
.await
.expect("bucket incarnation");
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-stage", fi.clone())
.await
.expect("stage reserved body");
let service = make_server_for_context(Some(fixture.context.clone()));
let mut reserve = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let mut stale = Request::new(reserve.get_ref().clone());
stale.get_mut().bucket_incarnation_id = Uuid::new_v4().as_bytes().to_vec().into();
let stale_body = rustfs_protos::canonical_metadata_reservation_request_body(stale.get_ref()).expect("stale body");
set_tonic_canonical_body_digest(&mut stale, &stale_body).expect("stale digest");
mark_v2_authenticated(&mut stale);
assert!(
!service
.reserve_object_metadata(stale)
.await
.expect("stale handler")
.into_inner()
.success,
"a token must not be issued for a different bucket incarnation"
);
let mut self_lock = Request::new(reserve.get_ref().clone());
self_lock.get_mut().object = rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_owned();
let self_lock_body =
rustfs_protos::canonical_metadata_reservation_request_body(self_lock.get_ref()).expect("self-lock body");
set_tonic_canonical_body_digest(&mut self_lock, &self_lock_body).expect("self-lock digest");
mark_v2_authenticated(&mut self_lock);
assert!(
!service
.reserve_object_metadata(self_lock)
.await
.expect("self-lock handler")
.into_inner()
.success,
"a token cannot hold the same lock that its object commit must acquire"
);
let body = rustfs_protos::canonical_metadata_reservation_request_body(reserve.get_ref()).expect("reserve body");
set_tonic_canonical_body_digest(&mut reserve, &body).expect("reserve digest");
mark_v2_authenticated(&mut reserve);
let response = service
.reserve_object_metadata(reserve)
.await
.expect("reserve RPC")
.into_inner();
assert!(response.success, "reservation must succeed: {:?}", response.error);
assert_eq!(response.protocol_version, 2);
let token = response.token;
let mut tampered_renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: token.clone(),
renew_only: false,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(tampered_renew.get_ref())
.expect("ordinary activation body");
set_tonic_canonical_body_digest(&mut tampered_renew, &body).expect("ordinary activation digest");
mark_v2_authenticated(&mut tampered_renew);
tampered_renew.get_mut().renew_only = true;
assert_eq!(
service
.activate_object_metadata_reservation(tampered_renew)
.await
.expect_err("changing activation into renewal must invalidate the body digest")
.code(),
tonic::Code::PermissionDenied
);
let mut renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: token.clone(),
renew_only: true,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renew body");
set_tonic_canonical_body_digest(&mut renew, &body).expect("renew digest");
mark_v2_authenticated(&mut renew);
assert!(
service
.activate_object_metadata_reservation(renew)
.await
.expect("renew RPC")
.into_inner()
.success,
"prepared reservation should renew without activation"
);
let mut activate = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: token.clone(),
renew_only: false,
});
let body =
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
mark_v2_authenticated(&mut activate);
let activated = service
.activate_object_metadata_reservation(activate)
.await
.expect("activate RPC")
.into_inner();
assert!(activated.success, "activation must succeed: {:?}", activated.error);
let message = ReservedRenameDataRequest {
rename: Some(RenameDataRequest {
disk: disk.endpoint().to_string(),
src_volume: RUSTFS_META_TMP_BUCKET.into(),
src_path: "reserved-stage".into(),
file_info: serde_json::to_string(&fi).expect("encode metadata"),
dst_volume: "target-rpc".into(),
dst_path: "destination".into(),
..Default::default()
}),
token,
};
let body = rustfs_protos::canonical_reserved_rename_data_request_body(&message).expect("rename body");
let mut tampered = Request::new(message.clone());
set_tonic_canonical_body_digest(&mut tampered, &body).expect("rename digest");
mark_v2_authenticated(&mut tampered);
tampered.get_mut().rename.as_mut().expect("rename").dst_path = "other".into();
assert_eq!(
service
.rename_reserved_data(tampered)
.await
.expect_err("tampered object")
.code(),
tonic::Code::PermissionDenied
);
let mut rename = Request::new(message);
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
mark_v2_authenticated(&mut rename);
let response = service.rename_reserved_data(rename).await.expect("rename RPC").into_inner();
assert!(response.success, "reserved rename must succeed: {:?}", response.error);
let read = disk
.read_version(
"target-rpc",
"target-rpc",
"destination",
&fi.version_id.expect("version").to_string(),
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read committed object");
assert_eq!(read.data, fi.data);
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial]
async fn reserved_rename_rpc_cancellation_does_not_stop_physical_publication() {
assert_reserved_rename_rpc_retains_fence_after_cancellation(false).await;
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial]
async fn short_reserved_rename_rpc_retains_fence_past_expiry_after_cancellation() {
assert_reserved_rename_rpc_retains_fence_after_cancellation(true).await;
}
#[cfg(not(windows))]
async fn assert_reserved_rename_rpc_retains_fence_after_cancellation(short: bool) {
use crate::storage::storage_api::{
LocalPublicationPause, LocalPublicationStage,
contract::namespace::NamespaceLocking as _,
ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions},
};
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("target-rpc", false).await;
let incarnation = fixture
.env
.ecstore
.bucket_incarnation_id("target-rpc")
.await
.expect("bucket incarnation");
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-cancel", fi.clone())
.await
.expect("stage reserved body");
let service = make_server_for_context(Some(fixture.context.clone()));
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener");
let addr = listener.local_addr().expect("RPC listener address");
let (shutdown, stopped) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
tonic::transport::Server::builder()
.add_service(NodeServiceServer::new(service))
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
let _ = stopped.await;
})
.await
.expect("serve RPC target");
});
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
.await
.expect("connect RPC client");
let mut prepare = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(prepare.get_ref()).expect("prepare body");
set_tonic_canonical_body_digest(&mut prepare, &body).expect("prepare digest");
mark_v2_authenticated(&mut prepare);
let prepared = if short {
client.reserve_object_metadata_short(prepare).await
} else {
client.reserve_object_metadata(prepare).await
}
.expect("prepare handler")
.into_inner();
assert!(prepared.success, "prepare: {:?}", prepared.error);
let mut renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: prepared.token.clone(),
renew_only: true,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renew body");
set_tonic_canonical_body_digest(&mut renew, &body).expect("renew digest");
mark_v2_authenticated(&mut renew);
assert!(
client
.activate_object_metadata_reservation(renew)
.await
.expect("renew handler")
.into_inner()
.success,
"renewal over the real transport must retain the prepared token"
);
let mut activate = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: prepared.token.clone(),
renew_only: false,
});
let body =
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
mark_v2_authenticated(&mut activate);
let activated = client
.activate_object_metadata_reservation(activate)
.await
.expect("activate handler")
.into_inner();
assert!(activated.success, "activate: {:?}", activated.error);
let mut rename = Request::new(ReservedRenameDataRequest {
rename: Some(RenameDataRequest {
disk: disk.endpoint().to_string(),
src_volume: RUSTFS_META_TMP_BUCKET.into(),
src_path: "reserved-cancel".into(),
dst_volume: "target-rpc".into(),
dst_path: "destination".into(),
file_info: serde_json::to_string(&fi).expect("encode metadata"),
..Default::default()
}),
token: prepared.token,
});
let body = rustfs_protos::canonical_reserved_rename_data_request_body(rename.get_ref()).expect("rename body");
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
mark_v2_authenticated(&mut rename);
let mut pause =
LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename)
.expect("install physical pause");
let mut handler = Box::pin(client.rename_reserved_data(rename));
super::timeout(Duration::from_secs(10), async {
tokio::select! {
result = &mut handler => panic!("handler completed before physical entry: {result:?}"),
entered = pause.entered() => entered.expect("physical executor entered"),
}
})
.await
.expect("bounded physical entry");
drop(handler);
assert!(
fixture.env.ecstore.namespace_commits_pending_for_test(),
"physical publication remains active"
);
if short {
// Once claimed, the physical write owns the fence independently
// of the now-expired unclaimed-token lease.
tokio::time::sleep(Duration::from_secs(3)).await;
}
let lifecycle = fixture
.env
.ecstore
.new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.expect("bucket publication lock");
let mut bucket_writer = Box::pin(lifecycle.get_write_lock(Duration::from_secs(10)));
let early = super::timeout(Duration::from_millis(500), &mut bucket_writer).await;
let old_lifecycle = fixture
.env
.ecstore
.new_ns_lock("target-rpc", "")
.await
.expect("bucket lifecycle lock");
let mut old_writer = Box::pin(old_lifecycle.get_write_lock(Duration::from_secs(10)));
let old_early = super::timeout(Duration::from_millis(500), &mut old_writer).await;
drop(pause);
match early {
Ok(result) => {
drop(result.expect("early lifecycle result"));
panic!("bucket publication writer acquired while the remote physical write was still paused");
}
Err(_) => drop(
super::timeout(Duration::from_secs(10), bucket_writer)
.await
.expect("lifecycle owner drains")
.expect("bucket writer acquires"),
),
}
match old_early {
Ok(result) => {
drop(result.expect("early lifecycle result"));
panic!("bucket lifecycle writer acquired while the remote physical write was still paused");
}
Err(_) => drop(
super::timeout(Duration::from_secs(10), old_writer)
.await
.expect("lifecycle owner drains")
.expect("bucket lifecycle writer acquires"),
),
}
super::timeout(Duration::from_secs(10), async {
while fixture.env.ecstore.namespace_commits_pending_for_test() {
tokio::task::yield_now().await;
}
})
.await
.expect("physical publication drains");
let read = disk
.read_version(
"target-rpc",
"target-rpc",
"destination",
&fi.version_id.expect("version").to_string(),
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read published object");
assert_eq!(read.data, fi.data);
drop(client);
let _ = shutdown.send(());
super::timeout(Duration::from_secs(10), server)
.await
.expect("RPC server shuts down")
.expect("RPC server task");
}
#[cfg(not(windows))]
#[tokio::test]
async fn quota_fence_handlers_commit_real_data_and_drain_before_revocation() {
@@ -8974,28 +9820,47 @@ mod tests {
))
}
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
async fn internal_snapshot_once(
root: &std::path::Path,
) -> std::io::Result<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
let mut snapshot = std::collections::BTreeMap::new();
let mut directories = (0..4)
.map(|index| std::path::PathBuf::from(format!("disk{index}/{RUSTFS_META_BUCKET}")))
.collect::<Vec<_>>();
while let Some(relative) = directories.pop() {
let mut entries = tokio::fs::read_dir(root.join(&relative))
.await
.expect("read internal snapshot directory");
let mut entries = tokio::fs::read_dir(root.join(&relative)).await?;
snapshot.insert(relative.clone(), None);
while let Some(entry) = entries.next_entry().await.expect("read internal snapshot entry") {
while let Some(entry) = entries.next_entry().await? {
let path = relative.join(entry.file_name());
let file_type = entry.file_type().await.expect("read internal snapshot entry type");
let file_type = entry.file_type().await?;
if file_type.is_dir() {
directories.push(path);
} else {
assert!(file_type.is_file(), "fixture snapshot must contain only directories and regular files");
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await.expect("read snapshot file bytes")));
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await?));
}
}
}
snapshot
Ok(snapshot)
}
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
// Startup cleanup can remove temporary paths after the namespace
// commit drains. Capture the complete tree only once it is stable.
super::timeout(Duration::from_secs(10), async {
let mut previous = None;
loop {
match internal_snapshot_once(root).await {
Ok(snapshot) if previous.as_ref() == Some(&snapshot) => return snapshot,
Ok(snapshot) => previous = Some(snapshot),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => previous = None,
Err(error) => panic!("read internal snapshot: {error}"),
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("internal snapshot must stabilize within ten seconds")
}
fn file_info(object: &str, version: Uuid, body: Bytes) -> FileInfo {
+254
View File
@@ -13,6 +13,76 @@
// limitations under the License.
use super::{LocalMutationTarget, NodeService};
#[cfg(test)]
pub(super) mod short_response_test_hook {
use std::sync::{LazyLock, Mutex};
use tokio::sync::oneshot;
use uuid::Uuid;
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Phase {
Reserve,
Renew,
Activate,
}
struct Hook {
id: Uuid,
bucket: String,
phase: Phase,
entered: oneshot::Sender<()>,
release: oneshot::Receiver<()>,
}
static HOOK: LazyLock<Mutex<Option<Hook>>> = LazyLock::new(|| Mutex::new(None));
pub(crate) struct Pause {
id: Uuid,
pub(crate) entered: oneshot::Receiver<()>,
release: Option<oneshot::Sender<()>>,
}
pub(crate) fn install(bucket: &str, phase: Phase) -> Pause {
let id = Uuid::new_v4();
let (entered_tx, entered) = oneshot::channel();
let (release, release_rx) = oneshot::channel();
let mut hook = HOOK.lock().expect("short response hook");
assert!(hook.is_none(), "short response hook already installed");
*hook = Some(Hook {
id,
bucket: bucket.to_owned(),
phase,
entered: entered_tx,
release: release_rx,
});
Pause {
id,
entered,
release: Some(release),
}
}
impl Drop for Pause {
fn drop(&mut self) {
HOOK.lock().expect("short response hook").take_if(|hook| hook.id == self.id);
if let Some(release) = self.release.take() {
let _ = release.send(());
}
}
}
pub(super) async fn wait(bucket: &str, phase: Phase) {
let hook = HOOK
.lock()
.expect("short response hook")
.take_if(|hook| hook.bucket == bucket && hook.phase == phase);
if let Some(hook) = hook {
let _ = hook.entered.send(());
let _ = hook.release.await;
}
}
}
use crate::storage::storage_api::ecstore_disk::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, POOL_META_NATIVE_CAS_METADATA_SUFFIX,
PoolMetaNativeRecoveryRequest as NativeRecoveryRequest, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET,
@@ -507,6 +577,190 @@ impl NodeService {
Ok(store)
}
pub(super) async fn handle_reserve_object_metadata(
&self,
request: Request<MetadataReservationRequest>,
short: bool,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reservation body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("metadata reservation requires body-bound authentication"))?;
let request = request.into_inner();
if request.timeout_ms == 0 {
return Err(Status::invalid_argument("reservation timeout must be positive"));
}
let incarnation = Uuid::from_slice(&request.bucket_incarnation_id)
.ok()
.filter(|id| !id.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid bucket incarnation"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
// A cancelled caller may never receive the token. Keep late v3
// acquisition bounded before the short unclaimed-token lease begins.
let max_timeout_ms = if short { 50 } else { 5000 };
let timeout = std::time::Duration::from_millis(request.timeout_ms.min(max_timeout_ms));
let result = if short {
store
.reserve_local_object_metadata_short(&request.disk, &request.bucket, &request.object, incarnation, timeout)
.await
} else {
store
.reserve_local_object_metadata_at_incarnation(
&request.disk,
&request.bucket,
&request.object,
incarnation,
timeout,
)
.await
};
#[cfg(test)]
if short && result.is_ok() {
short_response_test_hook::wait(&request.bucket, short_response_test_hook::Phase::Reserve).await;
}
let protocol_version = if short { 3 } else { 2 };
Ok(Response::new(match result {
Ok(token) => SnapshotLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
protocol_version,
error: None,
},
Err(error) => SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version,
error: Some(error.into()),
},
}))
}
pub(super) async fn handle_activate_object_metadata_reservation(
&self,
request: Request<MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reservation activation body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("reservation activation requires body-bound authentication"))?;
let request = request.into_inner();
let token = Uuid::from_slice(&request.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
let result = if request.renew_only {
store
.renew_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token)
.await
} else {
store
.activate_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token)
.await
};
#[cfg(test)]
if result.is_ok() {
let phase = if request.renew_only {
short_response_test_hook::Phase::Renew
} else {
short_response_test_hook::Phase::Activate
};
short_response_test_hook::wait(&request.bucket, phase).await;
}
Ok(Response::new(match result {
Ok(()) => SnapshotLeaseMutationResponse {
success: true,
error: None,
},
Err(error) => SnapshotLeaseMutationResponse {
success: false,
error: Some(error.into()),
},
}))
}
pub(super) async fn handle_cancel_object_metadata_reservation(
&self,
request: Request<MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reservation cancellation body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("reservation cancellation requires body-bound authentication"))?;
let request = request.into_inner();
let token = Uuid::from_slice(&request.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
Ok(Response::new(
match store.cancel_local_object_metadata_reservation(&request.disk, token).await {
Ok(cancelled) => SnapshotLeaseMutationResponse {
success: cancelled,
error: None,
},
Err(error) => SnapshotLeaseMutationResponse {
success: false,
error: Some(error.into()),
},
},
))
}
pub(super) async fn handle_rename_reserved_data(
&self,
request: Request<ReservedRenameDataRequest>,
) -> Result<Response<RenameDataResponse>, Status> {
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reserved rename body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("reserved rename requires body-bound authentication"))?;
let request = request.into_inner();
let token = Uuid::from_slice(&request.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
let rename = request.rename.ok_or_else(|| Status::invalid_argument("missing rename"))?;
if !rename.scanner_publication_lease_token.is_empty() || !rename.bucket_incarnation_id.is_empty() {
return Err(Status::invalid_argument("reserved rename does not support scanner or heal scope"));
}
let decoded = decode_rename_data_request_file_info(&rename.file_info_bin, &rename.file_info)
.map_err(|_| Status::invalid_argument("invalid reserved rename FileInfo"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
let result = store
.rename_local_data_reserved(
&rename.disk,
(&rename.src_volume, &rename.src_path),
&decoded.value,
(&rename.dst_volume, &rename.dst_path),
token,
)
.await;
let encoded = result.and_then(|value| encode_rename_data_response_payloads(&value, decoded.from_msgpack));
Ok(Response::new(match encoded {
Ok((rename_data_resp, rename_data_resp_bin)) => RenameDataResponse {
success: true,
rename_data_resp,
rename_data_resp_bin: rename_data_resp_bin.into(),
error: None,
},
Err(error) => RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Bytes::new(),
error: Some(error.into()),
},
}))
}
pub(super) async fn handle_acquire_snapshot_lease(
&self,
request: Request<SnapshotLeaseRequest>,
+194
View File
@@ -49,6 +49,91 @@ async fn refresh_lock(
lock_client: &std::sync::Arc<dyn rustfs_lock::client::LockClient>,
args: &LockRequest,
) -> GenerallyLockResponse {
let writer_marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
if args
.metadata
.tags
.get(writer_marker)
.is_some_and(|value| value == writer_marker)
{
return match lock_client.waiting_writer(&args.lock_id).await {
Ok(Some(waiting)) => GenerallyLockResponse {
success: true,
error_info: None,
lock_info: Some(format!("{writer_marker}:{}", u8::from(waiting))),
},
Ok(None) => GenerallyLockResponse {
success: false,
error_info: None,
lock_info: None,
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
let probe_marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
if args
.metadata
.tags
.get(probe_marker)
.is_some_and(|value| value == probe_marker)
{
return match lock_client.supports_heal_lock_transition(&args.lock_id).await {
Ok(success) => GenerallyLockResponse {
success,
error_info: None,
lock_info: success.then(|| probe_marker.to_owned()),
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
let upgrade_marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
if args
.metadata
.tags
.get(upgrade_marker)
.is_some_and(|value| value == upgrade_marker)
{
return match lock_client.try_upgrade_committed_tail(&args.lock_id).await {
Ok(success) => GenerallyLockResponse {
success,
error_info: None,
lock_info: success.then(|| upgrade_marker.to_owned()),
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
if args.metadata.tags.get(marker).is_some_and(|value| value == marker) {
return match lock_client.downgrade_committed_tail(&args.lock_id).await {
Ok(true) => GenerallyLockResponse {
success: true,
error_info: None,
lock_info: Some(marker.to_owned()),
},
Ok(false) => GenerallyLockResponse {
success: false,
error_info: None,
lock_info: None,
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
match lock_client.refresh(&args.lock_id).await {
Ok(true) => GenerallyLockResponse {
success: true,
@@ -347,6 +432,115 @@ mod tests {
assert!(result.lock_info.is_none());
}
#[tokio::test]
async fn refresh_reports_waiting_writer_for_held_lease() {
use super::refresh_lock;
use rustfs_lock::client::{HEAL_WAITING_WRITER_MARKER, LockClient};
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
use std::{sync::Arc, time::Duration};
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let key = ObjectKey::new("bucket", "writer-observation-rpc");
let mut held = LockRequest::new(key.clone(), LockType::Exclusive, "heal");
assert!(client.acquire_lock(&held).await.expect("acquire heal lease").success);
held.metadata
.tags
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
let writer = {
let client = Arc::clone(&client);
tokio::spawn(async move {
let request = LockRequest::new(key, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_secs(3));
let response = client.acquire_lock(&request).await.expect("writer request");
(request, response)
})
};
let expected = format!("{HEAL_WAITING_WRITER_MARKER}:1");
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let observed = refresh_lock(&client, &held).await;
if observed.lock_info.as_deref() == Some(expected.as_str()) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("waiting writer reported");
assert!(client.release(&held.lock_id).await.expect("release heal lease"));
let (request, response) = writer.await.expect("writer task");
assert!(response.success);
assert!(client.release(&request.lock_id).await.expect("release writer"));
}
#[tokio::test]
async fn refresh_upgrade_seals_lease_against_late_conversion() {
use super::refresh_lock;
use rustfs_lock::client::{
COMMITTED_TAIL_CONVERSION_MARKER, COMMITTED_TAIL_UPGRADE_MARKER, HEAL_LOCK_TRANSITION_PROBE_MARKER,
HEAL_WAITING_WRITER_MARKER, LockClient,
};
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
use std::sync::Arc;
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let mut request = LockRequest::new(ObjectKey::new("bucket", "upgrade-rpc"), LockType::Exclusive, "owner");
assert!(client.acquire_lock(&request).await.expect("acquire lease").success);
request
.metadata
.tags
.insert(HEAL_LOCK_TRANSITION_PROBE_MARKER.into(), HEAL_LOCK_TRANSITION_PROBE_MARKER.into());
let probe = refresh_lock(&client, &request).await;
assert!(probe.success);
assert_eq!(probe.lock_info.as_deref(), Some(HEAL_LOCK_TRANSITION_PROBE_MARKER));
request.metadata.tags.clear();
request
.metadata
.tags
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
let observation = refresh_lock(&client, &request).await;
assert!(observation.success);
assert_eq!(observation.lock_info.as_deref(), Some(format!("{HEAL_WAITING_WRITER_MARKER}:0").as_str()));
request.metadata.tags.clear();
request
.metadata
.tags
.insert(COMMITTED_TAIL_UPGRADE_MARKER.into(), COMMITTED_TAIL_UPGRADE_MARKER.into());
let response = refresh_lock(&client, &request).await;
assert!(response.success);
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_UPGRADE_MARKER));
request.metadata.tags.clear();
request
.metadata
.tags
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
assert!(!refresh_lock(&client, &request).await.success, "late conversion must be rejected");
assert!(client.release(&request.lock_id).await.expect("release lease"));
}
#[tokio::test]
async fn refresh_can_convert_committed_tail_and_acknowledges_protocol_version() {
use super::refresh_lock;
use rustfs_lock::client::{COMMITTED_TAIL_CONVERSION_MARKER, LockClient};
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
use std::{sync::Arc, time::Duration};
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let resource = ObjectKey::new("bucket", "tail-conversion-rpc");
let mut request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner");
assert!(client.acquire_lock(&request).await.expect("acquire write lease").success);
request
.metadata
.tags
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
let response = refresh_lock(&client, &request).await;
assert!(response.success);
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_CONVERSION_MARKER));
let read = LockRequest::new(resource, LockType::Shared, "reader").with_acquire_timeout(Duration::from_secs(1));
assert!(client.acquire_lock(&read).await.expect("read committed tail").success);
assert!(client.release(&read.lock_id).await.expect("release reader"));
assert!(client.release(&request.lock_id).await.expect("release tail"));
}
// B1 -- refresh_lock delegates to LocalClient::refresh for a held lock, returns
// success=true, and actually extends expires_at (inverts the "parse then drop" stub).
#[tokio::test]
+5 -61
View File
@@ -44,58 +44,45 @@ pub(crate) fn table_matches_staged_base(table: &TableEntry, commit_log: &CommitL
pub(crate) struct TableCommitHistoryIndex<'a> {
table_id: &'a str,
reachable_states: BTreeSet<(&'a str, &'a str)>,
reachable_version_tokens: BTreeSet<&'a str>,
committed_successors: BTreeMap<(&'a str, &'a str), &'a CommitLogEntry>,
ambiguous_states: BTreeSet<(&'a str, &'a str)>,
cycle_detected: bool,
version_token_reused: bool,
}
impl<'a> TableCommitHistoryIndex<'a> {
pub(crate) fn new(table: &'a TableEntry, commits: impl IntoIterator<Item = &'a CommitLogEntry>) -> Self {
let mut by_new_state = BTreeMap::<(&str, &str), Option<&CommitLogEntry>>::new();
let mut by_new_state = BTreeMap::<(&str, &str), Option<(&str, &str)>>::new();
let mut ambiguous_states = BTreeSet::new();
for commit in commits
.into_iter()
.filter(|commit| commit.table_id == table.table_id && !matches!(commit.status, CommitLogStatus::Failed))
{
let key = (commit.new_metadata_location.as_str(), commit.new_version_token.as_str());
let previous = (commit.previous_metadata_location.as_str(), commit.expected_version_token.as_str());
by_new_state
.entry(key)
.and_modify(|candidate| {
*candidate = None;
ambiguous_states.insert(key);
})
.or_insert(Some(commit));
.or_insert(Some(previous));
}
let mut reachable_states = BTreeSet::new();
let mut reachable_version_tokens = BTreeSet::new();
let mut committed_successors = BTreeMap::new();
let mut version_token_reused = false;
let mut state = (table.metadata_location.as_str(), table.version_token.as_str());
let cycle_detected = loop {
if !reachable_states.insert(state) {
break true;
}
version_token_reused |= !reachable_version_tokens.insert(state.1);
let Some(commit) = by_new_state.get(&state).copied().flatten() else {
let Some(Some(previous)) = by_new_state.get(&state) else {
break false;
};
let previous = (commit.previous_metadata_location.as_str(), commit.expected_version_token.as_str());
if matches!(commit.status, CommitLogStatus::Committed) {
committed_successors.insert(previous, commit);
}
state = previous;
state = *previous;
};
Self {
table_id: &table.table_id,
reachable_states,
reachable_version_tokens,
committed_successors,
ambiguous_states,
cycle_detected,
version_token_reused,
}
}
@@ -110,49 +97,6 @@ impl<'a> TableCommitHistoryIndex<'a> {
.reachable_states
.contains(&(target.new_metadata_location.as_str(), target.new_version_token.as_str()))
}
fn proven_superseding_commit(&self, target: &CommitLogEntry) -> Option<&CommitLogEntry> {
if self.cycle_detected
|| self.version_token_reused
|| !self.ambiguous_states.is_empty()
|| self.table_id != target.table_id
|| !matches!(target.status, CommitLogStatus::Staged)
|| self.reachable_version_tokens.contains(target.new_version_token.as_str())
{
return None;
}
let base = (target.previous_metadata_location.as_str(), target.expected_version_token.as_str());
self.committed_successors
.get(&base)
.copied()
.filter(|winner| winner.commit_id != target.commit_id)
}
}
/// Classifies an object-backed staged loser without rewriting its receipt.
///
/// A reachable finalized successor from the same pointer/token base proves that
/// the old request lost CAS. Its STAGED record and idempotency binding remain
/// available, and replay of that old request must still conflict. Missing or
/// contradictory proof remains a manual-review state.
pub(crate) fn table_commit_recovery_entry_from_history(
table: &TableEntry,
commit_log: &CommitLogEntry,
idempotency_commit: Option<&CommitLogEntry>,
history: &TableCommitHistoryIndex<'_>,
) -> TableCommitRecoveryEntry {
let mut entry = table_commit_recovery_entry(table, commit_log, idempotency_commit, history.proves_committed(commit_log));
if entry.recovery_state == TableCommitRecoveryState::ManualReview
&& matches!(
entry.idempotency_index_status,
TableCommitIdempotencyIndexStatus::NotRequired | TableCommitIdempotencyIndexStatus::Matches
)
&& let Some(winner) = history.proven_superseding_commit(commit_log)
{
entry.recovery_state = TableCommitRecoveryState::Superseded;
entry.reason = format!("staged request lost CAS to reachable committed successor {}", winner.commit_id);
}
entry
}
pub(crate) fn table_catalog_recovery_summary(
-1
View File
@@ -1415,7 +1415,6 @@ pub(crate) struct TableCatalogDiagnosticsReport {
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCommitRecoveryState {
Committed,
Superseded,
StagedBeforeTableUpdate,
FinalizationRequired,
IdempotencyIndexRepairRequired,
+8 -4
View File
@@ -3065,11 +3065,11 @@ where
}
None => None,
};
commits.push(table_commit_recovery_entry_from_history(
commits.push(table_commit_recovery_entry(
entry,
commit_log,
idempotency_commit.as_ref(),
&history,
history.proves_committed(commit_log),
));
}
commits.sort_by(|left, right| left.commit_id.cmp(&right.commit_id));
@@ -3168,8 +3168,12 @@ where
}
None => None,
};
let recovery_entry =
table_commit_recovery_entry_from_history(&entry, commit_log, idempotency_commit.as_ref(), &history);
let recovery_entry = table_commit_recovery_entry(
&entry,
commit_log,
idempotency_commit.as_ref(),
history.proves_committed(commit_log),
);
if matches!(
recovery_entry.recovery_state,
TableCommitRecoveryState::FinalizationRequired | TableCommitRecoveryState::IdempotencyIndexRepairRequired
-1
View File
@@ -20,7 +20,6 @@ use std::sync::Arc;
mod strong_maintenance_planning;
mod strong_maintenance_planning_paths;
mod superseded_commit_recovery;
const TABLE_CATALOG_TEST_TIMEOUT: StdDuration = StdDuration::from_secs(30);
@@ -1,487 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
struct StagedLoserFixture {
backend: TestCatalogObjectBackend,
store: ObjectTableCatalogStore<TestCatalogObjectBackend>,
loser: TableCommitRequest,
winner: TableCommitRequest,
committed: TableCommitResult,
}
async fn staged_loser_fixture(interrupt_winner_finalization: bool) -> StagedLoserFixture {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let initial = test_table_entry(
bucket,
&namespace,
&table,
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
);
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &namespace))
.await
.unwrap();
store.create_table(initial.clone()).await.unwrap();
for name in ["00001", "loser", "winner", "reloaded"] {
let location = default_table_metadata_file_path(&namespace, &table, &format!("{name}.metadata.json"));
let mut metadata = super::super::test_support::table_metadata_json(&initial.table_uuid, &initial.warehouse_location);
metadata["properties"] = serde_json::json!({"writer": name});
backend
.seed_object(bucket, &location, serde_json::to_vec(&metadata).unwrap())
.await;
}
let paths = TableCatalogObjectPaths::default();
let table_path = paths.table_entry_path(bucket, &namespace, &table);
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
let loser = TableCommitRequest {
table_bucket: bucket.to_string(),
namespace: namespace.public_name(),
table: table.as_str().to_string(),
commit_id: "paused-writer-a0".to_string(),
idempotency_key: Some("a0-request".to_string()),
operation: "set-properties".to_string(),
expected_version_token: initial.version_token.clone(),
expected_metadata_location: initial.metadata_location.clone(),
new_metadata_location: default_table_metadata_file_path(&namespace, &table, "loser.metadata.json"),
requirements: Vec::new(),
writer: None,
};
assert_matches!(store.commit_table(loser.clone()).await, Err(TableCatalogStoreError::Internal(_)));
let winner = TableCommitRequest {
commit_id: "winner-c1".to_string(),
idempotency_key: Some("c1-request".to_string()),
new_metadata_location: default_table_metadata_file_path(&namespace, &table, "winner.metadata.json"),
..loser.clone()
};
if interrupt_winner_finalization {
let winner_log = paths.commit_log_entry_path(bucket, &initial.table_id, &winner.commit_id);
backend.fail_put_attempt(RUSTFS_META_BUCKET, &winner_log, 2).await;
}
let committed = store.commit_table(winner.clone()).await.unwrap();
StagedLoserFixture {
backend,
store,
loser,
winner,
committed,
}
}
#[tokio::test]
async fn superseded_staged_commit_recovers_without_rewriting_receipts_or_pointer() {
let StagedLoserFixture {
backend,
store,
loser,
winner,
committed,
} = staged_loser_fixture(false).await;
let bucket = loser.table_bucket.as_str();
let before = backend.state.lock().await.objects.clone();
let recovered = store.recover_table_commits(bucket, "sales", "orders").await.unwrap();
assert_eq!(recovered.manual_review_count, 0, "a durable same-base winner proves the staged loser");
assert_eq!(recovered.finalized_count, 0, "a losing writer must never be finalized as committed");
let lost = recovered
.commits
.iter()
.find(|entry| entry.commit_id == loser.commit_id)
.unwrap();
assert_eq!(lost.status, CommitLogStatus::Staged);
assert_eq!(serde_json::to_value(&lost.recovery_state).unwrap(), "SUPERSEDED");
assert_eq!(lost.idempotency_index_status, TableCommitIdempotencyIndexStatus::Matches);
let after = backend.state.lock().await.objects.clone();
assert_eq!(after.len(), before.len());
for (path, original) in &before {
let actual = after.get(path).unwrap();
assert_eq!(actual.data, original.data, "recovery changed bytes at {path:?}");
assert_eq!(actual.etag, original.etag, "recovery rewrote {path:?}");
}
let restarted = ObjectTableCatalogStore::new(backend.clone());
let diagnostics = restarted.diagnose_table_catalog(bucket, "sales", "orders", 0).await.unwrap();
assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::Healthy);
assert_matches!(restarted.commit_table(loser.clone()).await, Err(TableCatalogStoreError::Conflict(_)));
let mut stale_request = loser.clone();
stale_request.commit_id = "stale-new-request".to_string();
stale_request.idempotency_key = Some("stale-new-key".to_string());
assert_matches!(restarted.commit_table(stale_request).await, Err(TableCatalogStoreError::Conflict(_)));
let mut different_payload = loser.clone();
different_payload.operation = "replace".to_string();
assert_matches!(
restarted.commit_table(different_payload.clone()).await,
Err(TableCatalogStoreError::Conflict(_))
);
assert_eq!(
restarted.load_table(bucket, "sales", "orders").await.unwrap(),
Some(committed.table.clone())
);
assert_eq!(restarted.commit_table(winner).await.unwrap().table, committed.table);
let reloaded = TableCommitRequest {
commit_id: "reloaded-writer-a1".to_string(),
idempotency_key: Some("a1-request".to_string()),
expected_version_token: committed.table.version_token.clone(),
expected_metadata_location: committed.table.metadata_location.clone(),
new_metadata_location: default_table_metadata_file_path(
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
"reloaded.metadata.json",
),
..different_payload
};
let next = restarted.commit_table(reloaded.clone()).await.unwrap();
assert_eq!(next.table.generation, committed.table.generation + 1);
assert_eq!(restarted.commit_table(reloaded).await.unwrap().table, next.table);
let restarted = ObjectTableCatalogStore::new(backend);
assert_eq!(
restarted
.diagnose_table_catalog(bucket, "sales", "orders", 0)
.await
.unwrap()
.recovery_status,
TableCatalogRecoveryStatus::Healthy
);
}
#[tokio::test]
async fn superseded_staged_commit_waits_for_durable_winner_finalization() {
let StagedLoserFixture {
backend,
store,
loser,
committed,
..
} = staged_loser_fixture(true).await;
let paths = TableCatalogObjectPaths::default();
let log_path = paths.commit_log_entry_path(&loser.table_bucket, &committed.table.table_id, &loser.commit_id);
let before = backend.read_object(RUSTFS_META_BUCKET, &log_path).await.unwrap().unwrap();
let planned = store
.plan_table_commit_recovery("analytics", "sales", "orders")
.await
.unwrap();
assert_eq!(planned.manual_review_count, 1, "a merely staged winner is insufficient proof");
assert_eq!(planned.finalization_required_count, 1);
let recovered = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
assert_eq!(recovered.finalized_count, 1);
assert_eq!(recovered.manual_review_count, 0);
let loser_state = recovered
.commits
.iter()
.find(|entry| entry.commit_id == loser.commit_id)
.unwrap();
assert_eq!(loser_state.recovery_state, TableCommitRecoveryState::Superseded);
assert_eq!(loser_state.status, CommitLogStatus::Staged);
let after = backend.read_object(RUSTFS_META_BUCKET, &log_path).await.unwrap().unwrap();
assert_eq!(after.data, before.data);
assert_eq!(after.etag, before.etag);
assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap(), Some(committed.table));
assert_eq!(
store
.diagnose_table_catalog("analytics", "sales", "orders", 0)
.await
.unwrap()
.recovery_status,
TableCatalogRecoveryStatus::Healthy
);
}
#[tokio::test]
async fn superseded_staged_commit_requires_retained_winner_proof_after_restart() {
let StagedLoserFixture {
backend,
store,
loser,
winner,
committed,
} = staged_loser_fixture(false).await;
let healthy = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
assert_eq!(healthy.manual_review_count, 0);
let winner_path =
TableCatalogObjectPaths::default().commit_log_entry_path("analytics", &committed.table.table_id, &winner.commit_id);
assert!(
backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), winner_path))
.is_some()
);
let restarted = ObjectTableCatalogStore::new(backend);
let before = restarted.load_table("analytics", "sales", "orders").await.unwrap();
let report = restarted.recover_table_commits("analytics", "sales", "orders").await.unwrap();
assert_eq!(report.manual_review_count, 1);
assert_eq!(report.finalized_count, 0);
assert_eq!(report.commits[0].recovery_state, TableCommitRecoveryState::ManualReview);
assert_eq!(report.commits[0].status, CommitLogStatus::Staged);
assert_matches!(restarted.commit_table(loser).await, Err(TableCatalogStoreError::Conflict(_)));
assert_eq!(restarted.load_table("analytics", "sales", "orders").await.unwrap(), before);
let diagnostics = restarted
.diagnose_table_catalog("analytics", "sales", "orders", 0)
.await
.unwrap();
assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::ManualReviewRequired);
assert!(
diagnostics
.recommended_actions
.contains(&TableCatalogRecoveryAction::ReviewCommitLog)
);
}
#[tokio::test]
async fn superseded_staged_commit_does_not_repair_a_conflicting_index() {
let StagedLoserFixture {
backend,
store,
loser,
committed,
..
} = staged_loser_fixture(false).await;
let mut conflicting = store
.get_commit_by_id("analytics", &committed.table.table_id, &loser.commit_id)
.await
.unwrap()
.unwrap();
conflicting.new_version_token = "contradictory-token".to_string();
let path = TableCatalogObjectPaths::default().commit_idempotency_entry_path(
"analytics",
&committed.table.table_id,
loser.idempotency_key.as_deref().unwrap(),
);
backend
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&conflicting).unwrap())
.await;
let before = backend.state.lock().await.objects.clone();
let report = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
assert_eq!(report.manual_review_count, 1);
assert_eq!(report.finalized_count, 0);
let loser_state = report
.commits
.iter()
.find(|entry| entry.commit_id == loser.commit_id)
.unwrap();
assert_eq!(loser_state.recovery_state, TableCommitRecoveryState::ManualReview);
assert_eq!(loser_state.idempotency_index_status, TableCommitIdempotencyIndexStatus::Conflicting);
let after = backend.state.lock().await.objects.clone();
assert_eq!(before.len(), after.len());
for (key, original) in &before {
let actual = after.get(key).unwrap();
assert_eq!(actual.data, original.data);
assert_eq!(actual.etag, original.etag);
}
assert_matches!(store.commit_table(loser).await, Err(TableCatalogStoreError::Conflict(_)));
}
fn history_fixture() -> (TableEntry, CommitLogEntry, CommitLogEntry) {
let namespace = Namespace::parse("sales").unwrap();
let table_name = IdentifierSegment::parse("orders").unwrap();
let metadata = |name| default_table_metadata_file_path(&namespace, &table_name, name);
let mut table = test_table_entry("analytics", &namespace, &table_name, metadata("winner.metadata.json"));
table.version_token = "winner-token".to_string();
table.generation = 2;
let loser = CommitLogEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
commit_id: "loser".to_string(),
idempotency_key: None,
table_id: table.table_id.clone(),
operation: "set-properties".to_string(),
expected_version_token: "base-token".to_string(),
new_version_token: "loser-token".to_string(),
previous_metadata_location: metadata("base.metadata.json"),
new_metadata_location: metadata("loser.metadata.json"),
requirements: Vec::new(),
status: CommitLogStatus::Staged,
writer: None,
created_at: None,
updated_at: None,
};
let winner = CommitLogEntry {
commit_id: "winner".to_string(),
new_version_token: table.version_token.clone(),
new_metadata_location: table.metadata_location.clone(),
status: CommitLogStatus::Committed,
..loser.clone()
};
(table, loser, winner)
}
fn classify_history(
table: &TableEntry,
target: &CommitLogEntry,
commits: &[CommitLogEntry],
indexed: Option<&CommitLogEntry>,
) -> TableCommitRecoveryEntry {
let history = TableCommitHistoryIndex::new(table, commits.iter());
table_commit_recovery_entry_from_history(table, target, indexed, &history)
}
#[test]
fn superseded_history_requires_a_reachable_finalized_same_base_winner() {
for case in [
"staged-winner",
"failed-winner",
"wrong-base-token",
"wrong-base-location",
"foreign-table",
"disconnected-winner",
"same-commit-id",
] {
let (mut table, loser, mut winner) = history_fixture();
match case {
"staged-winner" => winner.status = CommitLogStatus::Staged,
"failed-winner" => winner.status = CommitLogStatus::Failed,
"wrong-base-token" => winner.expected_version_token = "other-base-token".to_string(),
"wrong-base-location" => winner.previous_metadata_location = "other/base.metadata.json".to_string(),
"foreign-table" => winner.table_id = "other-table-id".to_string(),
"disconnected-winner" => {
table.metadata_location = "other/current.metadata.json".to_string();
table.version_token = "other-current-token".to_string();
}
"same-commit-id" => winner.commit_id.clone_from(&loser.commit_id),
_ => unreachable!(),
}
let recovery = classify_history(&table, &loser, &[loser.clone(), winner], None);
assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview, "{case}");
}
}
#[test]
fn superseded_history_is_stable_after_later_commits() {
let (mut table, loser, winner) = history_fixture();
let immediate = classify_history(&table, &loser, &[loser.clone(), winner.clone()], None);
assert_eq!(immediate.recovery_state, TableCommitRecoveryState::Superseded);
assert!(immediate.reason.contains(&winner.commit_id));
let later = CommitLogEntry {
commit_id: "later".to_string(),
previous_metadata_location: table.metadata_location.clone(),
expected_version_token: table.version_token.clone(),
new_metadata_location: "later/current.metadata.json".to_string(),
new_version_token: "later-token".to_string(),
..winner.clone()
};
table.metadata_location.clone_from(&later.new_metadata_location);
table.version_token.clone_from(&later.new_version_token);
table.generation += 1;
let after = classify_history(&table, &loser, &[loser.clone(), winner, later], None);
assert_eq!(after.recovery_state, TableCommitRecoveryState::Superseded);
assert_eq!(after.reason, immediate.reason);
}
#[test]
fn superseded_history_rejects_failed_or_committed_losers() {
for status in [CommitLogStatus::Failed, CommitLogStatus::Committed] {
let (table, mut loser, winner) = history_fixture();
loser.status = status;
let recovery = classify_history(&table, &loser, &[loser.clone(), winner], None);
assert_eq!(recovery.recovery_state, TableCommitRecoveryState::ManualReview);
}
}
#[test]
fn superseded_history_rejects_ambiguity_cycles_and_reused_tokens() {
for case in [
"ambiguous-current",
"cycle",
"reused-history-token",
"loser-token-in-history",
"off-history-ambiguity",
] {
let (mut table, mut loser, winner) = history_fixture();
let mut extra = CommitLogEntry {
commit_id: "extra".to_string(),
..winner.clone()
};
let mut commits = vec![loser.clone(), winner.clone()];
match case {
"ambiguous-current" => commits.push(extra),
"cycle" => {
extra.previous_metadata_location.clone_from(&winner.new_metadata_location);
extra.expected_version_token.clone_from(&winner.new_version_token);
extra.new_metadata_location.clone_from(&winner.previous_metadata_location);
extra.new_version_token.clone_from(&winner.expected_version_token);
commits.push(extra);
}
"reused-history-token" => {
extra.previous_metadata_location.clone_from(&winner.new_metadata_location);
extra.expected_version_token.clone_from(&winner.new_version_token);
extra.new_metadata_location = "later/current.metadata.json".to_string();
extra.new_version_token.clone_from(&winner.expected_version_token);
table.metadata_location.clone_from(&extra.new_metadata_location);
table.version_token.clone_from(&extra.new_version_token);
commits.push(extra);
}
"loser-token-in-history" => {
loser.new_version_token.clone_from(&winner.expected_version_token);
commits[0] = loser.clone();
}
"off-history-ambiguity" => {
extra.new_metadata_location = "unknown/metadata.json".to_string();
extra.new_version_token = "unknown-token".to_string();
commits.push(extra.clone());
extra.commit_id = "another-extra".to_string();
commits.push(extra);
}
_ => unreachable!(),
}
assert_eq!(
classify_history(&table, &loser, &commits, None).recovery_state,
TableCommitRecoveryState::ManualReview,
"{case}"
);
}
}
#[test]
fn superseded_history_requires_an_intact_idempotency_binding() {
let (table, mut loser, winner) = history_fixture();
loser.idempotency_key = Some("loser-request".to_string());
let commits = [loser.clone(), winner];
assert_eq!(
classify_history(&table, &loser, &commits, Some(&loser)).recovery_state,
TableCommitRecoveryState::Superseded
);
let missing = classify_history(&table, &loser, &commits, None);
assert_eq!(missing.recovery_state, TableCommitRecoveryState::ManualReview);
assert_eq!(missing.idempotency_index_status, TableCommitIdempotencyIndexStatus::Missing);
let mut stale = loser.clone();
stale.status = CommitLogStatus::Committed;
let state = classify_history(&table, &loser, &commits, Some(&stale));
assert_eq!(state.recovery_state, TableCommitRecoveryState::ManualReview);
assert_eq!(state.idempotency_index_status, TableCommitIdempotencyIndexStatus::Stale);
let mut conflicting = loser.clone();
conflicting.commit_id = "another-request".to_string();
let state = classify_history(&table, &loser, &commits, Some(&conflicting));
assert_eq!(state.recovery_state, TableCommitRecoveryState::ManualReview);
assert_eq!(state.idempotency_index_status, TableCommitIdempotencyIndexStatus::Conflicting);
}
#[test]
fn superseded_history_never_relabels_a_committed_staged_ancestor() {
let (table, mut ancestor, winner) = history_fixture();
ancestor.new_metadata_location.clone_from(&winner.previous_metadata_location);
ancestor.new_version_token.clone_from(&winner.expected_version_token);
ancestor.previous_metadata_location = "older/base.metadata.json".to_string();
ancestor.expected_version_token = "older-base-token".to_string();
let recovery = classify_history(&table, &ancestor, &[ancestor.clone(), winner], None);
assert_eq!(recovery.recovery_state, TableCommitRecoveryState::FinalizationRequired);
}
+11 -27
View File
@@ -540,7 +540,7 @@ struct CleanupObservation {
elapsed: Duration,
}
async fn observe_cleanup(replies: [CleanupReply; 2], measurement_delay: Duration) -> CleanupObservation {
async fn observe_cleanup(replies: [CleanupReply; 2]) -> CleanupObservation {
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
const PRIVATE_BODY: &str = "private-cleanup-response-body";
@@ -583,12 +583,6 @@ async fn observe_cleanup(replies: [CleanupReply; 2], measurement_delay: Duration
.to_owned();
let reply = if index < 2 {
assert!(request_line.starts_with("PUT "));
if index == 1 && !measurement_delay.is_zero() {
// Model descheduling during the measured PUT without a wall-clock sleep.
tokio::time::pause();
tokio::time::advance(measurement_delay).await;
tokio::time::resume();
}
CleanupReply::Status(200)
} else {
assert!(request_line.starts_with("DELETE "));
@@ -622,8 +616,6 @@ async fn observe_cleanup(replies: [CleanupReply; 2], measurement_delay: Duration
)
.expect("cleanup probe");
let mut request = request(ObjectOperation::PutObject);
// Status fixtures must reach cleanup independently of short measurement deadlines.
request.duration = REAL_SERVER_MEASUREMENT_BUDGET;
request.traffic_bytes = 4;
let log = CleanupLog::default();
let writer = log.clone();
@@ -715,7 +707,7 @@ async fn cleanup_diagnostics_identify_each_failed_http_stage_without_private_con
([CleanupReply::Status(204), CleanupReply::Status(403)], "bucket_delete", 403),
([CleanupReply::Status(204), CleanupReply::Status(404)], "bucket_delete", 404),
] {
let observed = observe_cleanup(replies, Duration::ZERO).await;
let observed = observe_cleanup(replies).await;
assert_cleanup_failure(&observed, stage, "CleanupFailed", Some(status));
assert_eq!(observed.deletes.len(), if stage == "object_delete" { 1 } else { 2 });
}
@@ -728,7 +720,7 @@ async fn cleanup_diagnostics_identify_each_disconnected_transport_stage() {
([CleanupReply::Disconnect, CleanupReply::Status(204)], "object_delete"),
([CleanupReply::Status(204), CleanupReply::Disconnect], "bucket_delete"),
] {
let observed = observe_cleanup(replies, Duration::ZERO).await;
let observed = observe_cleanup(replies).await;
assert_cleanup_failure(&observed, stage, "ProtocolFailure", None);
}
}
@@ -736,21 +728,13 @@ async fn cleanup_diagnostics_identify_each_disconnected_transport_stage() {
#[tokio::test]
async fn cleanup_accepts_missing_object_and_still_deletes_the_bucket() {
let _guard = TEST_LOCK.lock().await;
for measurement_delay in [Duration::ZERO, Duration::from_secs(2)] {
let observed = observe_cleanup([CleanupReply::Status(404), CleanupReply::Status(204)], measurement_delay).await;
assert_eq!(
observed.measurement.result.outcome(),
ObjectOutcome::Succeeded,
"target result: {:?}; cleanup diagnostics: {}",
observed.measurement.target,
observed.log
);
assert_eq!(observed.measurement.target.reason_code, ObjectTargetReasonCode::Complete);
assert_eq!(observed.deletes.len(), 2);
assert!(observed.deletes[0].ends_with("/synthetic-object HTTP/1.1"));
assert!(observed.deletes[1].ends_with("/ HTTP/1.1"));
assert!(!observed.log.contains("connect_object_cleanup_failed"));
}
let observed = observe_cleanup([CleanupReply::Status(404), CleanupReply::Status(204)]).await;
assert_eq!(observed.measurement.result.outcome(), ObjectOutcome::Succeeded);
assert_eq!(observed.measurement.target.reason_code, ObjectTargetReasonCode::Complete);
assert_eq!(observed.deletes.len(), 2);
assert!(observed.deletes[0].ends_with("/synthetic-object HTTP/1.1"));
assert!(observed.deletes[1].ends_with("/ HTTP/1.1"));
assert!(!observed.log.contains("connect_object_cleanup_failed"));
}
#[tokio::test]
@@ -766,7 +750,7 @@ async fn cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_dead
"bucket_delete",
),
] {
let observed = observe_cleanup(replies, Duration::ZERO).await;
let observed = observe_cleanup(replies).await;
assert_cleanup_failure(&observed, stage, "TimedOut", None);
assert!(observed.elapsed >= Duration::from_secs(5));
assert!(observed.log.contains("remaining_ms=0"), "{}", observed.log);
+5 -10
View File
@@ -8,7 +8,6 @@ KEYCLOAK_IMAGE="${KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:6a7217a100bd3
WORK_DIR="${RUNNER_TEMP:-/tmp}/rustfs-keycloak-live-${$}"
KEYCLOAK_CONTAINER="rustfs-keycloak-live-${$}"
RUSTFS_PID=""
DOCKER=(docker)
mkdir -p "${WORK_DIR}"
cleanup() {
@@ -18,10 +17,10 @@ cleanup() {
wait "${RUSTFS_PID}" 2>/dev/null || true
fi
if [[ "${status}" -ne 0 ]]; then
"${DOCKER[@]}" logs "${KEYCLOAK_CONTAINER}" >"${WORK_DIR}/keycloak.log" 2>&1 || true
docker logs "${KEYCLOAK_CONTAINER}" >"${WORK_DIR}/keycloak.log" 2>&1 || true
echo "live-gate logs retained in ${WORK_DIR}" >&2
fi
"${DOCKER[@]}" rm -f "${KEYCLOAK_CONTAINER}" >/dev/null 2>&1 || true
docker rm -f "${KEYCLOAK_CONTAINER}" >/dev/null 2>&1 || true
if [[ "${status}" -eq 0 ]]; then
rm -rf "${WORK_DIR}"
fi
@@ -34,10 +33,6 @@ for command in curl docker python3 aws; do
exit 1
}
done
if ! docker info >/dev/null 2>&1; then
DOCKER=(sudo -n docker)
"${DOCKER[@]}" info >/dev/null
fi
[[ -x "${RUSTFS_BINARY}" ]] || {
echo "RustFS binary is not executable: ${RUSTFS_BINARY}" >&2
exit 1
@@ -64,7 +59,7 @@ ISSUER="${KEYCLOAK_ORIGIN}/realms/rustfs-ci"
DISCOVERY_URL="${ISSUER}/.well-known/openid-configuration"
RUSTFS_ORIGIN="http://127.0.0.1:${RUSTFS_PORT}"
"${DOCKER[@]}" run --detach --rm \
docker run --detach --rm \
--name "${KEYCLOAK_CONTAINER}" \
--memory 1g \
--publish "127.0.0.1:${KEYCLOAK_PORT}:8080" \
@@ -79,8 +74,8 @@ for _ in $(seq 1 120); do
if curl --noproxy '*' -fsS "${DISCOVERY_URL}" >"${WORK_DIR}/discovery.json" 2>/dev/null; then
break
fi
if [[ "$("${DOCKER[@]}" inspect -f '{{.State.Running}}' "${KEYCLOAK_CONTAINER}" 2>/dev/null || true)" != "true" ]]; then
"${DOCKER[@]}" logs "${KEYCLOAK_CONTAINER}" >&2 || true
if [[ "$(docker inspect -f '{{.State.Running}}' "${KEYCLOAK_CONTAINER}" 2>/dev/null || true)" != "true" ]]; then
docker logs "${KEYCLOAK_CONTAINER}" >&2 || true
echo "Keycloak exited before discovery became ready" >&2
exit 1
fi
-120
View File
@@ -220,126 +220,6 @@ SH
self.assertNotIn(" continue-on-error: true", self.publish)
self.assertNotIn(" overwrite: true", upload)
def dispatch_shell(self):
step = self.steps["Dispatch nightly candidate to auto-testing"]
start = step.index(" run: |") + 1
return "\n".join(line[10:] for line in step[start:] if not line.strip() or line.startswith(" "))
def candidate_manifest(self, **overrides):
digest = overrides.pop("package_sha256", self.digest)
workflow_sha = overrides.pop("workflow_sha", self.sha)
manifest = {
"schema": 2,
"workflow_sha": workflow_sha,
"source_ref": "main",
"source_sha": self.sha,
"build_run_id": 12345,
"build_run_attempt": 1,
"package_sha256": digest,
"package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{digest}/rustfs.deb",
}
manifest.update(overrides)
return manifest
def run_dispatch(self, manifest=None, **overrides):
manifest = self.candidate_manifest() if manifest is None else manifest
candidate = self.root / "candidate.json"
candidate.write_text(json.dumps(manifest))
shim = self.root / "dispatch-curl.sh"
shim.write_text(r'''curl() {
printf '%s\0' "$@" > "$FAKE_CURL_ARGS"
local outfile="" data=""
local args=("$@")
local i=0
while [[ $i -lt ${#args[@]} ]]; do
case "${args[$i]}" in
-o) outfile="${args[$((i+1))]}"; i=$((i+2));;
--data-binary)
data="${args[$((i+1))]}"
cp "${data#@}" "$FAKE_DISPATCH_BODY"
i=$((i+2));;
*) i=$((i+1));;
esac
done
[[ "$FAKE_DISPATCH_STATUS" != "curl-failure" ]] || return 28
if [[ -n "$outfile" ]]; then printf '%s' "$FAKE_DISPATCH_RESPONSE" > "$outfile"; fi
printf '%s' "$FAKE_DISPATCH_STATUS"
}
''')
env = dict(os.environ, BASH_ENV=str(shim), CANDIDATE_FILE=str(candidate), DEB_DATE="2026-09-06",
SOURCE_REPO="rustfs/rustfs", DISPATCH_TOKEN="test-dispatch-token", RUNNER_TEMP=str(self.root),
GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_SHA=manifest["workflow_sha"],
FAKE_CURL_ARGS=str(self.root / "curl-args"), FAKE_DISPATCH_BODY=str(self.root / "dispatch-body.json"),
FAKE_DISPATCH_STATUS="204", FAKE_DISPATCH_RESPONSE="")
env.update(overrides)
return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.dispatch_shell()],
cwd=self.root, env=env, capture_output=True, text=True)
def test_dispatch_follows_release_publish_without_blocking_vault_lanes(self):
names = list(self.steps)
self.assertLess(names.index("Publish packages to auto-testing release assets"), names.index("Dispatch nightly candidate to auto-testing"))
step = "\n".join(self.steps["Dispatch nightly candidate to auto-testing"])
self.assertIn(" if: ${{ steps.publish.outputs.candidate_file != '' }}", step)
self.assertIn(" DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}", step)
self.assertNotIn("secrets.GITHUB_TOKEN", step)
self.assertNotIn("github.token", step)
self.assertNotIn("continue-on-error", step)
self.assertIn("https://api.github.com/repos/rustfs/auto-testing/dispatches", step)
self.assertIn("nightly-candidate-ready", step)
lines = WORKFLOW.read_text().splitlines()
for lane in ("kms-vault-lane", "kms-vault-ha-failover"):
job = yaml_block(lines, lane, 2)
needs = next(line for line in job if line.strip().startswith("needs:"))
self.assertEqual(needs.strip(), "needs: resolve-source")
self.assertNotIn("Dispatch nightly candidate", "\n".join(job))
alert = "\n".join(yaml_block(lines, "alert-on-failure", 2))
self.assertIn("needs: [build, kms-vault-lane, kms-vault-ha-failover]", alert)
chain = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
self.assertIn('workflows: ["Nightly GNU Build"]', chain)
self.assertIn("types: [completed]", chain)
def test_dispatch_sends_schema2_candidate_without_the_token(self):
result = self.run_dispatch()
self.assertEqual(result.returncode, 0, result.stderr)
body = json.loads((self.root / "dispatch-body.json").read_text())
self.assertEqual(body["event_type"], "nightly-candidate-ready")
payload = body["client_payload"]
self.assertLessEqual(len(payload), 10)
self.assertEqual(set(payload), {"event_type", "source_repo", "latest_alias_url", "auto_testing_release", "channel", "deb_date", "nightly_workflow", "candidate"})
self.assertEqual(payload["candidate"], self.candidate_manifest())
self.assertEqual(payload["latest_alias_url"], "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb")
self.assertEqual(payload["auto_testing_release"], "https://github.com/rustfs/auto-testing/releases/tag/nightly")
self.assertEqual(payload["channel"], "nightly")
self.assertEqual(payload["deb_date"], "2026-09-06")
self.assertEqual(payload["nightly_workflow"], ".github/workflows/nightly-gnu.yml")
self.assertEqual(payload["source_repo"], "rustfs/rustfs")
validate_manifest(payload["candidate"], {"id": 12345, "run_attempt": 1, "head_sha": self.sha})
args = (self.root / "curl-args").read_bytes().split(b"\0")
self.assertIn(b"https://api.github.com/repos/rustfs/auto-testing/dispatches", args)
self.assertIn(b"Authorization: Bearer test-dispatch-token", args)
self.assertNotIn("test-dispatch-token", json.dumps(body))
self.assertIn("dispatched nightly-candidate-ready", result.stdout)
def test_dispatch_failure_stops_before_treating_the_candidate_as_notified(self):
for status, response in (("422", '{"message":"No more than 10 properties are allowed"}'), ("500", "unavailable"), ("curl-failure", "")):
with self.subTest(status=status):
result = self.run_dispatch(FAKE_DISPATCH_STATUS=status, FAKE_DISPATCH_RESPONSE=response)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("repository_dispatch", result.stderr)
if response:
self.assertIn(response, result.stderr)
self.assertNotIn("dispatched nightly-candidate-ready", result.stdout)
for marker in ("curl-args", "dispatch-body.json"):
(self.root / marker).unlink(missing_ok=True)
missing = self.run_dispatch(DISPATCH_TOKEN="")
self.assertNotEqual(missing.returncode, 0)
self.assertIn("AUTO_TESTING_DISPATCH_TOKEN is required", missing.stderr)
self.assertFalse((self.root / "curl-args").exists())
rejected = self.run_dispatch(manifest=self.candidate_manifest(extra="nope"))
self.assertNotEqual(rejected.returncode, 0, rejected.stdout)
self.assertIn("nightly-candidate-ready contract", rejected.stderr)
self.assertFalse((self.root / "dispatch-body.json").exists())
class ValidationPackageTests(unittest.TestCase):
def setUp(self):
-41
View File
@@ -261,47 +261,6 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
if missing:
self.assertIn(f"missing required STS field: {missing}", result.stderr)
def test_oidc_docker_access_and_cleanup_use_the_same_privileges(self):
tools = self.directory / "docker-tools"
tools.mkdir()
(tools / "docker").write_text(
'#!/bin/bash\n'
'printf "%s %s\\n" "${DOCKER_PRIVILEGED:-no}" "$*" >> "$DOCKER_CALLS"\n'
'case "$1" in\n'
' info) [ "$DOCKER_MODE" = direct ] || '
'{ [ "$DOCKER_MODE" = sudo ] && [ "${DOCKER_PRIVILEGED:-no}" = yes ]; } ;;\n'
' run) exit 43 ;;\n'
' *) exit 0 ;;\n'
'esac\n'
)
(tools / "sudo").write_text(
'#!/bin/bash\n[ "$1" = -n ] || exit 92\nshift\n'
'export DOCKER_PRIVILEGED=yes\nexec "$@"\n'
)
for name in ("docker", "sudo"):
(tools / name).chmod(0o755)
(tools / "python3").symlink_to(sys.executable)
for name in ("aws", "curl"):
(tools / name).symlink_to("/usr/bin/true")
for mode in ("direct", "sudo", "unavailable"):
with self.subTest(mode=mode):
calls = self.directory / f"docker-{mode}.log"
result = subprocess.run(
["bash", str(ROOT / "scripts/test/oidc_keycloak_live.sh"), "/usr/bin/true"],
env={**os.environ, "PATH": f"{tools}:/usr/bin:/bin", "RUNNER_TEMP": str(self.directory),
"DOCKER_MODE": mode, "DOCKER_CALLS": str(calls)},
capture_output=True, text=True, timeout=10,
)
log = calls.read_text()
if mode == "unavailable":
self.assertNotEqual(result.returncode, 0)
self.assertNotIn(" run ", log)
else:
self.assertEqual(result.returncode, 43, result.stderr)
privilege = "yes" if mode == "sudo" else "no"
for command in ("run", "logs", "rm -f"):
self.assertIn(f"{privilege} {command} ", log)
def test_existing_evidence_directory_is_rejected(self) -> None:
for suffix in ("", "-scratch"):
self.setUp()