Compare commits

..

2 Commits

Author SHA1 Message Date
hector e494dc2b4b ci(validation): sweep the migrated canonical validation cases nightly (#9107)
* ci(validation): sweep the migrated canonical validation cases nightly

The canonical release-validation cases migrated into auto-testing by
rustfs/auto-testing#113 (66 implemented, all non-heal) currently run
only through the manual validation flow; the last fleet executions
stopped on 2026-10-27 and nothing schedules them. Add a scheduled
validation sweep that closes the gap:

- nightly at 19:30 UTC (after the functional chain settles), rotating
  windows of the implemented catalog cases so the whole catalog is
  covered roughly weekly, with a bounded batch per night
- workflow_dispatch for a single case (--case), one module (--module),
  or an explicit batch (--limit), mirroring how the lanes are testable
  standalone
- fleet admission through functional_environment enter/finish with the
  same quarantine discipline as the lanes, inter-case node cleanup,
  per-case logs + reports retained as a sweep artifact
- a markdown sweep report uploaded to rustfs/dashboard under
  functional-reports/validation/ so the cases appear next to the lane
  reports, plus an alert-on-failure job for scheduled runs
- heal.* cases stay on the interactive validation flow: many need the
  reviewed native/tool collectors built by separate validation-tools
  runs before fleet execution

The registry gains the workflow responsibility (monitor: workflow,
max-age 60h so the watchdog notices a dead sweep) and the wiring test
baseline count moves with it.

Verified locally: check_wiring, check_test_wiring, test_validation_registry
(9/9), test_functional_chain + test_functional_chain_health +
test_security_workflow (99/99).

* ci(validation): group the sweep report by catalog module

The validation cases cover distinct areas (pool lifecycle, ILM
expiration, bucket/site replication, kms, data services, identity,
s3 compatibility), so a flat case table hides the category breakdown.
Render one section per catalog module with its own case table and a
Subtotal line, and keep the single grand - Total:/- PASS:/- FAIL:
block for the dashboard overview parser. Subtotals deliberately use
a different bullet prefix so the overview regexes cannot mistake a
module subtotal for the grand total.

* ci(validation): sweep the eight planned cases whose executors exist

The planned catalog entries are not all unimplemented: all eight
(kms.sse-kms-round-trip, tiering.transition-and-restore,
events.delivery, audit.request-correlation, iam.policy-enforcement,
oidc.login-and-session, s3.basic-object-round-trip and
scanner.usage-convergence) have dedicated executors routed in
run_validation_case.sh and live PASS evidence recorded in the
migration map. Widen the sweep filter to include them: the rotating
nightly batch now covers 74 cases (66 implemented + these 8), and the
fresh evidence each sweep produces feeds their acceptance review.
heal.* procedure-ready cases remain excluded - their executors need
the reviewed native/tool collectors built by separate
validation-tools runs before fleet execution.

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
2026-11-01 21:53:00 +08:00
Hauser Zhi 2b7c927ee8 perf(s3): streamline conditional multipart preflight (#9099)
* perf(s3): streamline conditional multipart preflight

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(e2e): isolate mc mirror fixture in full suite

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(connect): isolate cleanup deadline fixtures

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* test(ecstore): avoid lock cleanup refcount race

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(ecstore): defer decommission lookup lock loss

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-01 18:59:57 +08:00
47 changed files with 832 additions and 7812 deletions
+11 -5
View File
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
[[profile.default.overrides]]
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
threads-required = "num-test-threads"
# This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
test-group = 'embedded-test-ports'
[[profile.ci.overrides]]
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
threads-required = "num-test-threads"
# Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -970,10 +970,16 @@ threads-required = "num-test-threads"
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
test-group = 'e2e-vault'
# Keep storage-heavy cases isolated in the full suite too. The large pagination
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
# Keep the storage-heavy smoke cases isolated in the full suite too. They
# start multiple server fixtures and issue many sequential S3 operations.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
threads-required = "num-test-threads"
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
# Reserve the runner so other server fixtures do not starve its disk operations.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
threads-required = "num-test-threads"
# Fixed, small contract for every executable server change. These existing
+26
View File
@@ -296,6 +296,32 @@
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
"name": "RustFS Validation Sweep",
"roles": [
"periodic",
"manual"
],
"events": [
"schedule",
"workflow_dispatch"
],
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"30 19 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 60,
"result_owner": "rustfs-validation-sweep.yml",
"disposition": "canonical validation cases executed in rotating batches",
"failure_owner": "workflow",
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/btf-prototype-reference.yml",
"name": "BTF prototype reference",
@@ -0,0 +1,421 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: RustFS Validation Sweep
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
# on the shared functional fleet. Everything with a lane executor is swept in
# rotating nightly batches so the whole set is covered roughly weekly; a
# single case or one module can be dispatched manually. That is the
# implemented catalog (66 today) plus the eight planned cases whose dedicated
# executors already exist and have live PASS evidence - fresh nightly
# evidence feeds their acceptance review. heal.* cases stay on the
# interactive validation flow: they require reviewed native/tool collectors
# built by separate validation-tools runs before fleet execution.
on:
schedule:
# After the nightly functional chain settles (chain starts ~16:48 UTC and
# runs for about two hours).
- cron: "30 19 * * *"
workflow_dispatch:
inputs:
case:
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
type: string
required: false
default: ''
module:
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
type: string
required: false
default: ''
limit:
description: 'Maximum cases per sweep (0 = run everything selected)'
type: number
required: false
default: 12
build_run_id:
description: 'Successful nightly package run; empty resolves the latest'
type: string
required: false
default: ''
build_run_attempt:
description: 'Attempt of that package run'
type: string
required: false
default: '1'
permissions:
contents: read
concurrency:
group: rustfs-validation-sweep
cancel-in-progress: false
env:
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
jobs:
sweep:
if: github.repository == 'rustfs/rustfs'
runs-on: smoke-testing
timeout-minutes: 150
steps:
- name: Checkout chain tooling
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Resolve nightly candidate
id: candidate
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
BUILD_RUN_ID: ${{ inputs.build_run_id }}
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
run: |
set -euo pipefail
python3 scripts/resolve_functional_candidate.py
python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
root.mkdir(parents=True, exist_ok=True)
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
manifest = candidate["manifest"]
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
output.write("package_url=" + manifest["package_url"] + "\n")
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
- name: Checkout auto-testing scripts
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ steps.candidate.outputs.testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Select sweep cases
id: select
env:
SELECTED_CASE: ${{ inputs.case }}
SELECTED_MODULE: ${{ inputs.module }}
CASE_LIMIT: ${{ inputs.limit }}
SWEEP_EVENT: ${{ github.event_name }}
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -euo pipefail
python3 - <<'PY'
import datetime
import json
import os
from pathlib import Path
sweep_root = Path(os.environ["SWEEP_ROOT"])
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
def sweepable(case):
status = case.get("source_status")
if status == "implemented":
return True
# The remaining planned non-heal cases route to the lane
# executors and already have live PASS evidence; sweeping them
# nightly produces fresh evidence for their acceptance review.
return status == "planned" and not case["id"].startswith("heal.")
selected = [(case["id"], module["id"]) for module in catalog["modules"]
for case in module["cases"]
if sweepable(case)]
if not selected:
raise SystemExit("catalog has no sweepable cases")
one_case = os.environ.get("SELECTED_CASE", "").strip()
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
known = {case_id for case_id, _ in selected}
modules = {module_id for _, module_id in selected}
if one_case:
if one_case not in known:
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
selected = [(one_case, dict(selected)[one_case])]
elif module_filter:
if module_filter not in modules:
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
selected = [pair for pair in selected if pair[1] == module_filter]
elif os.environ.get("SWEEP_EVENT") == "schedule":
# Stateless rotation: every night take one window of the catalog
# so a full sweep needs len(selected)/limit nights and no state
# has to be carried between runs.
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
start = (day * limit) % len(selected)
selected = (selected[start:] + selected[:start])[:limit]
limit = int(os.environ.get("CASE_LIMIT") or 0)
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
if limit > 0:
selected = selected[:limit]
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write(f"count={len(selected)}\n")
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
PY
- name: Admit clean shared environment
id: environment
timeout-minutes: 5
env:
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
run: python3 scripts/functional_environment.py enter --suite heal
- name: Run validation cases
id: run_cases
timeout-minutes: 120
env:
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -uo pipefail
mkdir -p "${SWEEP_ROOT}/results"
: > "${SWEEP_ROOT}/results.txt"
while read -r CASE_ID; do
[ -n "${CASE_ID}" ] || continue
echo "::group::case ${CASE_ID}"
rm -rf "${SWEEP_ROOT}/run"
if ./auto-testing/scripts/run_validation_case.sh \
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
echo "case ${CASE_ID}: PASS"
else
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
fi
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
fi
echo "::endgroup::"
# Fleet hygiene between cases: the case runners set up their own
# topology, this purge guarantees the next case starts clean even
# when the previous one died mid-run.
for node in ${RUSTFS_NODES}; do
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
${SUDO} dpkg -P rustfs
fi
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
${SUDO} rm -f /etc/default/rustfs
' || echo "inter-case cleanup failed on ${node} (continuing)"
done
done < "${SWEEP_ROOT}/sweep-cases.txt"
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
- name: Generate sweep report
if: ${{ always() && steps.environment.outcome == 'success' }}
env:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["SWEEP_ROOT"])
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
rows = []
for line in (root / "results.txt").read_text().splitlines():
if not line:
continue
case_id, status = line.split("\t")
rows.append((case_id, module_of.get(case_id, "?"), status))
passed = sum(1 for _, _, s in rows if s == "PASS")
failed = len(rows) - passed
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
f"{os.environ.get('GITHUB_RUN_ID', '')}")
report = [
"# RustFS validation sweep report", "",
f"- Run: {run_url}",
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
]
# Group the executed cases by catalog module (the major category:
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
# data services, identity, s3 compatibility). Per-module subtotals
# deliberately use "- Subtotal:" so the dashboard overview regexes
# (which anchor on "- Total:") keep reading the grand totals below.
by_module = {}
module_order = []
for case_id, module, status in rows:
if module not in by_module:
by_module[module] = []
module_order.append(module)
by_module[module].append((case_id, status))
report += ["## Case Summary", ""]
for module in module_order:
cases = by_module[module]
mod_pass = sum(1 for _, st in cases if st == "PASS")
mod_fail = len(cases) - mod_pass
report += [f"### {module} ({len(cases)})", "",
"| Case | Status |", "| --- | --- |"]
report += [f"| {case_id} | {status} |" for case_id, status in cases]
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
report += [
"## Totals", "",
f"- Total: {len(rows)}",
f"- PASS: {passed}",
f"- FAIL: {failed}",
"- UNSUPPORTED: 0", "",
"Failing cases keep this workflow red only as a report; the sweep artifact",
"retains every case log and per-case report for diagnosis.", "",
]
(root / "report.md").write_text("\n".join(report))
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
PY
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
- name: Upload functional report to dashboard
if: ${{ always() && steps.environment.outcome == 'success' }}
timeout-minutes: 2
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
SUITE: validation
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
exit 0
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
echo "dashboard updated: ${REPORT_PATH}"
- name: Retain sweep evidence
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/validation-sweep/report.md
${{ runner.temp }}/validation-sweep/results.txt
${{ runner.temp }}/validation-sweep/sweep-cases.txt
${{ runner.temp }}/validation-sweep/candidate.json
${{ runner.temp }}/validation-sweep/results/
if-no-files-found: error
retention-days: 30
- name: Cleanup environment (after)
id: environment_cleanup
timeout-minutes: 5
if: ${{ always() && steps.environment.outcome == 'success' }}
run: |
set -uo pipefail
cleanup_status=0
for node in ${RUSTFS_NODES}; do
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
set -euo pipefail
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
${SUDO} systemctl stop rustfs 2>/dev/null || true
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
${SUDO} dpkg -P rustfs
fi
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
${SUDO} rm -f /etc/default/rustfs
' || cleanup_status=$?
done
exit "${cleanup_status}"
- name: Release environment only after successful cleanup
if: ${{ always() && steps.environment.outcome == 'success' }}
timeout-minutes: 5
env:
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
- name: Notify on failure
if: failure()
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
- name: Enforce sweep result
if: ${{ always() && steps.environment.outcome == 'success' }}
env:
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
run: |
set -euo pipefail
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
echo "validation sweep has failing cases:"
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
exit 1
fi
alert-on-failure:
name: Alert on scheduled failure
# A scheduled sweep failure usually means a canonical validation case
# regressed or the fleet environment broke; surface it as a tracking
# issue instead of leaving it in the Actions tab.
needs: [sweep]
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
@@ -576,34 +576,6 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
async fn reserve_object_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn activate_object_metadata_reservation(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn cancel_object_metadata_reservation(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_reserved_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_data_with_pool_meta_cas(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
+7 -271
View File
@@ -67,15 +67,14 @@ use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
MetadataReservationCancelRequest, MetadataReservationRequest,
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -2453,212 +2452,6 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
}
impl RemoteDisk {
/// Prepare a remote token without retrying the RPC. It owns the bucket
/// publication fence but does not enter the object metadata queue yet.
pub async fn reserve_object_metadata(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: Duration,
) -> Result<Uuid> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
self.execute_with_timeout_for_op_and_health_action(
"reserve_object_metadata",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationRequest {
disk: self.endpoint.to_string(),
bucket: bucket.to_owned(),
object: object.to_owned(),
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
.map_err(|_| Error::other("reservation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.reserve_object_metadata(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
if response.protocol_version != 2 {
return Err(DiskError::UnsupportedDisk);
}
Uuid::from_slice(&response.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or(DiskError::Unexpected)
},
timeout.min(Duration::from_secs(5)),
FailureHealthAction::IgnoreFailure,
)
.await
}
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
/// result before allowing any subsequent unprotected mutation.
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
self.update_object_metadata_reservation(bucket, object, token, timeout, false)
.await
}
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
self.update_object_metadata_reservation(bucket, object, token, timeout, true)
.await
}
async fn update_object_metadata_reservation(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: Duration,
renew_only: bool,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
self.execute_with_timeout_for_op_and_health_action(
"activate_object_metadata_reservation",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationActivateRequest {
disk: self.endpoint.to_string(),
bucket: bucket.to_owned(),
object: object.to_owned(),
token: token.as_bytes().to_vec().into(),
renew_only,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
.map_err(|_| Error::other("reservation activation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.activate_object_metadata_reservation(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
Ok(())
},
timeout.min(Duration::from_secs(5)),
FailureHealthAction::IgnoreFailure,
)
.await
}
/// Cancel an unclaimed remote reservation. False cannot prove that an
/// earlier claim did not start a physical mutation.
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
self.execute_with_timeout_for_op(
"cancel_object_metadata_reservation",
|| async {
let mut client = self.get_client().await?;
let mut request = Request::new(MetadataReservationCancelRequest {
disk: self.endpoint.to_string(),
token: token.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.cancel_object_metadata_reservation(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if let Some(error) = response.error {
return Err(error.into());
}
Ok(response.success)
},
get_max_timeout_duration(),
)
.await
}
/// Publish a staged user-object write using a remote one-shot reservation.
/// The caller must retain the bucket lifecycle fence until the tail drains.
pub async fn rename_data_with_metadata_reservation(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
token: Uuid,
) -> Result<RenameDataResp> {
self.execute_with_timeout_for_op(
"rename_reserved_data",
|| async {
let mut client = self.get_client().await?;
let rename = RenameDataRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_owned(),
src_path: src_path.to_owned(),
file_info: compat_json(fi)?,
dst_volume: dst_volume.to_owned(),
dst_path: dst_path.to_owned(),
file_info_bin: encode_file_info_msgpack(fi)?.into(),
scanner_publication_lease_token: Bytes::new(),
bucket_incarnation_id: Bytes::new(),
};
let mut request = Request::new(ReservedRenameDataRequest {
rename: Some(rename),
token: token.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
.map_err(|_| Error::other("reserved rename request length overflow"))?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
let response = client
.rename_reserved_data(request)
.await
.map_err(|status| {
if status.code() == Code::Unimplemented {
DiskError::UnsupportedDisk
} else {
status.into()
}
})?
.into_inner();
if !response.success {
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
}
decode_msgpack_or_json::<RenameDataResp>(
&response.rename_data_resp_bin,
&response.rename_data_resp,
"RenameDataResp",
)
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed(
&self,
@@ -5316,63 +5109,6 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
runtime_sources::ensure_test_rpc_secret();
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
.await
.expect("old peer must bind");
let remote = remote_disk_for_addr(&peer.addr).await;
let token = Uuid::new_v4();
assert!(matches!(
remote
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.renew_object_metadata("bucket", "object", token, Duration::from_secs(5))
.await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote.cancel_object_metadata_reservation(token).await,
Err(DiskError::UnsupportedDisk)
));
assert!(matches!(
remote
.rename_data_with_metadata_reservation(
crate::disk::RUSTFS_META_TMP_BUCKET,
"staged",
&sample_rename_data_file_info(),
"bucket",
"object",
token,
)
.await,
Err(DiskError::UnsupportedDisk)
));
assert_eq!(
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
vec![
"/node_service.NodeService/ReserveObjectMetadata",
"/node_service.NodeService/ActivateObjectMetadataReservation",
"/node_service.NodeService/ActivateObjectMetadataReservation",
"/node_service.NodeService/CancelObjectMetadataReservation",
"/node_service.NodeService/RenameReservedData",
],
"reservation operations must never downgrade to legacy methods",
);
peer.stop().await;
}
#[tokio::test]
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
runtime_sources::ensure_test_rpc_secret();
@@ -1315,122 +1315,6 @@ impl LockClient for RemoteClient {
Ok(resp.success)
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
// Older nodes refresh normally but return no conversion marker.
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
if !response.success {
return Ok(None);
}
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
Some(":0") => Some(false),
Some(":1") => Some(true),
_ => None,
})
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
+154 -23
View File
@@ -3720,33 +3720,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
}
/// The storage error of a version copy that kept timing out on a namespace
/// lock through its in-place retries, kept typed so the entry failure policy
/// defers it.
/// A copy whose object namespace lock remained unavailable through its
/// in-place retries, retained as a typed deferral.
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct DecommissionCopyLockTimeout(#[source] Error);
struct DecommissionCopyLockUnavailable(#[source] Error);
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
match source {
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
Error::NamespaceLockQuorumUnavailable {
mode: "read",
bucket: lock_bucket,
object: lock_object,
..
} => {
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
&& lock_bucket == bucket
&& lock_object == object
}
_ => false,
}
}
/// A version copy that timed out acquiring a namespace lock leaves the source
/// A version copy that could not acquire its namespace lock leaves the source
/// in place; the next attempt copies the version again or finds an equivalent
/// committed target. The lock is held by another mutation of the key or, when
/// this node restarted mid-copy, by its previous incarnation until that lock's
/// lease expires on the peers. Either clears by waiting, so the entry is
/// deferred instead of being counted as a permanent per-object failure.
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
data_movement::data_movement_context_error(
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
DecommissionCopyLockTimeout(err),
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
DecommissionCopyLockUnavailable(err),
)
}
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
}
/// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3756,8 +3768,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
}
error
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
}
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5822,13 +5834,13 @@ enum DecommissionEntryOutcome {
/// entry is done and must not be counted as a failure or escalated. A quorum
/// failure of the source cleanup is deferred too: the copied source is still
/// in place and the next attempt cleans it up again. So is a version copy that
/// lost drive quorum or timed out on a namespace lock through its in-place
/// lost drive quorum or could not acquire an object namespace lock through its in-place
/// retries: the source still holds the version and the next attempt copies it
/// again. Every other failure stays terminal.
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
if is_decommission_source_cleanup_retryable_error(&err)
|| is_decommission_copy_quorum_deferral(&err)
|| is_decommission_copy_lock_timeout_deferral(&err)
|| is_decommission_copy_lock_unavailable_deferral(&err)
{
return Ok(DecommissionEntryOutcome::Deferred(err));
}
@@ -28353,7 +28365,7 @@ mod tests {
"one version must commit before the next version times out"
);
assert!(
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
"{outcome:?}"
);
assert_eq!(
@@ -28387,6 +28399,88 @@ mod tests {
assert_eq!(actual, b"written after versioning was enabled");
}
/// A target lookup that loses its read-lock quorum must retain the source
/// and resume an entry after an earlier version has already committed.
#[tokio::test]
#[serial_test::serial]
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
use tokio::io::AsyncReadExt as _;
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
let object = fixture.object;
let attempted = Arc::new(AtomicUsize::new(0));
let fault = DecommissionTestFaultGuard::install_with_error(
Arc::new({
let attempted = Arc::clone(&attempted);
move |stage, _, faulted, _, success| {
stage == "before_target_commit"
&& faulted == object
&& success
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
}
}),
Some(Arc::new(|_, bucket, object| {
data_movement::data_movement_stage_error_for_test(
"decommission_object",
"prepare_put_object",
bucket,
object,
Error::NamespaceLockQuorumUnavailable {
mode: "read",
bucket: bucket.to_owned(),
object: object.to_owned(),
required: 1,
achieved: 0,
},
)
})),
);
let (outcome, _) = fixture.run_entry().await;
assert!(
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
"{outcome:?}"
);
fixture.assert_source_retained().await;
assert_eq!(
fixture
.target_versions()
.await
.expect("first version committed")
.versions
.len(),
1
);
assert_eq!(
fixture.store.pool_meta.read().await.pools[0]
.decommission
.as_ref()
.expect("active decommission")
.items_decommission_failed,
0
);
drop(fault);
let (outcome, _) = fixture.run_entry().await;
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
fixture.assert_moved().await;
let mut reader = fixture.store.pools[1]
.get_object_reader(
fixture.bucket,
fixture.object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
..Default::default()
},
)
.await
.expect("the newer version must remain readable");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
assert_eq!(actual, b"written after versioning was enabled");
}
/// A namespace lock still held on the target key, as by a copy whose node
/// was killed mid-write, makes target selection fail with a typed
/// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28429,7 +28523,7 @@ mod tests {
let (outcome, _) = fixture.run_entry().await;
assert!(
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
"{outcome:?}"
);
assert_eq!(
@@ -28838,17 +28932,17 @@ mod tests {
timeout(),
);
for err in [timeout(), prepare] {
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
let mut error = Some(err);
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
.expect("a lock timeout copy must be deferred");
assert!(error.is_none());
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
assert!(!is_decommission_copy_quorum_deferral(&deferred));
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
.expect("the deferred error must keep its lock-timeout origin");
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
assert!(matches!(
resolve_decommission_entry_failure(deferred),
Ok(DecommissionEntryOutcome::Deferred(_))
@@ -28873,6 +28967,43 @@ mod tests {
}
}
#[test]
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
mode,
bucket: bucket.to_owned(),
object: object.to_owned(),
required: 1,
achieved: 0,
};
let wrap = |stage, err| {
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
};
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
.expect("matching read-lock quorum must defer");
assert!(error.is_none());
assert!(matches!(
resolve_decommission_entry_failure(deferred),
Ok(DecommissionEntryOutcome::Deferred(_))
));
for err in [
quorum("read", "bucket-a", "object-a"),
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
wrap(
"prepare_put_object",
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
),
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
] {
let mut error = Some(err);
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
assert!(error.is_some(), "unrelated lock failures must stay terminal");
}
}
#[test]
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
-13
View File
@@ -655,19 +655,6 @@ impl Sets {
.await
}
pub(crate) async fn put_object_with_prepared_reservations(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
self.get_disks_by_key(object)
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
.await
}
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
&self,
bucket: &str,
+17 -1
View File
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
#[derive(Debug)]
struct DataMovementStageError {
rendered: String,
stage: Option<String>,
source: Box<dyn std::error::Error + Send + Sync>,
}
@@ -626,6 +627,7 @@ where
{
Error::other(DataMovementStageError {
rendered,
stage: None,
source: Box::new(err),
})
}
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
where
E: std::error::Error + Send + Sync + 'static,
{
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
Error::other(DataMovementStageError {
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
stage: Some(stage.to_owned()),
source: Box::new(err),
})
}
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
let Error::Io(io_err) = err else {
return false;
};
io_err
.get_ref()
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
.is_some_and(|source| source.stage.as_deref() == Some(stage))
}
#[cfg(test)]
+21 -281
View File
@@ -604,69 +604,8 @@ impl LocalDiskWrapper {
let super::RenameDataGuards {
external_guard,
namespace_owner,
metadata_reservation,
metadata_reservation_token,
..
} = guards;
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
(Some(reservation), None) => Some(reservation),
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
Ok(reservation) => Some(reservation),
// A missing token may already have been claimed by an earlier
// request that is still publishing; do not claim it was safe.
Err(DiskError::DiskOngoingReq) => {
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
}
Err(error) => return super::RenameDataObservation::not_dispatched(error),
},
(None, None) => None,
};
// A queued reservation must acquire its metadata lease before disk
// admission. Otherwise a waiting background mutation could hold the
// disk permit needed by the earlier lease owner.
let metadata_reservation = match metadata_reservation {
Some(reservation) => Some(reservation.wait().await),
None => None,
};
// The claimed reservation stays in this cancellable future while disk
// admission can wait. Only physical work moves into an owned task.
let prechecked_permit = if metadata_reservation.is_some() {
// Background admission is exclusive. Waiting for it while holding
// this object's lease can deadlock an admitted mutation waiting for
// the same lease. An already owned permit does not need to wait.
if current_disk_io_context().is_some_and(|context| {
context.class == DiskIoClass::Background
&& !context
.permit
.as_ref()
.is_some_and(|permit| permit.is_for(&self.io_admission))
}) {
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
}
let preflight = self
.with_native_resources(async {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.acquire_disk_io_permit().await
})
.await;
match preflight {
Ok(Ok(Some(permit))) => Some(permit),
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
}
} else {
None
};
// The reserved metadata lease must outlive a cancelled RPC waiter.
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
Some(Arc::new(external_guard))
} else {
external_guard
};
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
@@ -685,54 +624,27 @@ impl LocalDiskWrapper {
};
let observed = run_owned_mutation(external_guard, move || async move {
let mut preflight_rejection = None;
let rename = || async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed = if let Some(reservation) = metadata_reservation {
Box::pin(operation.disk.rename_data_observed_with_reservation(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
(namespace_owner, Some(reservation)),
))
.await
} else {
Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
))
.await
};
preflight_rejection = observed.preflight_rejection;
observed.result
};
let result = if let Some(permit) = prechecked_permit {
with_disk_io_permit(Some(permit), async {
operation
.with_native_resources(operation.track_admitted_disk_operation(
"rename_data",
move |io_permit| async move {
let _io_permit = io_permit;
rename().await
},
Duration::ZERO,
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::Write,
true,
let result = operation
.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
// Preserve the former DiskAPI future's single boxing boundary.
let observed = Box::pin(operation.disk.rename_data_observed(
&src_volume,
&src_path,
&fi,
&dst_volume,
&dst_path,
namespace_owner,
))
.await?
})
.await
} else {
operation
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
.await
};
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
},
timeout_duration,
)
.await;
// Health tracking must observe the real disk error, not an Ok tuple.
Ok(super::RenameDataObservation {
result,
@@ -1686,67 +1598,6 @@ pub struct LocalDiskWrapper {
}
impl LocalDiskWrapper {
/// Reserve the metadata mutation lock for one object on this disk instance.
/// Unclaimed reservations expire after 60 seconds.
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.reserve_object_metadata(bucket, object, timeout).await
}
/// Register an object metadata reservation in FIFO order on this disk.
/// The returned token may be claimed before its physical lease is available.
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.enqueue_object_metadata(bucket, object).await
}
/// Prepare an inert reservation token bound to this disk and object.
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
self.prepare_object_metadata_with_owner(bucket, object, None).await
}
pub(crate) async fn prepare_object_metadata_with_owner(
&self,
bucket: &str,
object: &str,
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
) -> Result<Uuid> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
}
/// Refresh an inert token while the caller is still staging the write.
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.renew_object_metadata(bucket, object, token)
}
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
self.check_disk_stale().await?;
self.disk.activate_object_metadata(bucket, object, token).await
}
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
self.disk.cancel_object_metadata_reservation(token)
}
/// Create a new LocalDiskWrapper
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
@@ -2730,7 +2581,6 @@ impl LocalDiskWrapper {
timeout_duration,
timeout_health_action,
mutation,
false,
))
.await?
}
@@ -2742,7 +2592,6 @@ impl LocalDiskWrapper {
timeout_duration: Duration,
timeout_health_action: TimeoutHealthAction,
mutation: DiskMetricMutation,
identity_prechecked: bool,
) -> Result<T>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
@@ -2764,7 +2613,7 @@ impl LocalDiskWrapper {
}
// Check if disk is stale
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
if let Err(err) = self.check_disk_stale().await {
self.metrics.record_availability_error();
return Err(err);
}
@@ -5732,115 +5581,6 @@ mod tests {
drop(metadata_lease);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
let dir = tempfile::tempdir().expect("disk directory");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let meta = dir.path().join(RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
.await
.expect("write format");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
disk.make_volume("bucket").await.expect("bucket volume");
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
let token = wrapper
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
.await
.expect("reserve key");
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
disk.format_info.write().await.last_check = None;
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
disk.install_format_probe_hook(false, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv_timeout(Duration::from_secs(30));
});
let publisher = Arc::clone(&wrapper);
let rename = tokio::spawn(async move {
publisher
.rename_data_observed_with_guards(
RUSTFS_META_TMP_BUCKET,
"staging",
&FileInfo::default(),
"bucket",
"object",
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), entered_rx)
.await
.expect("disk identity preflight starts")
.expect("preflight hook");
rename.abort();
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
let key = disk.get_object_path("bucket", "object").expect("object path");
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
release_tx.send(()).expect("release disk identity preflight");
if !released_before_io {
drop(
tokio::time::timeout(Duration::from_secs(10), successor)
.await
.expect("preflight finishes"),
);
}
assert!(
released_before_io,
"a cancelled request must not retain the reservation before physical mutation starts"
);
}
#[tokio::test]
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
let dir = tempfile::tempdir().expect("disk directory");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
let token = wrapper
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
.await
.expect("reserve key");
let held = wrapper
.io_admission
.background(&CancellationToken::new())
.await
.expect("background permit");
let fi = FileInfo::default();
let rename = crate::disk::with_background_disk_io(
CancellationToken::new(),
wrapper.rename_data_observed_with_guards(
RUSTFS_META_TMP_BUCKET,
"staging",
&fi,
"bucket",
"object",
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
),
);
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
drop(held);
let observation = result.expect("reserved mutation must not wait while holding the object lease");
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
assert!(observation.rejected_before_publication());
}
#[tokio::test]
async fn delete_versions_counts_faulty_drive_availability_rejection() {
let dir = tempfile::tempdir().expect("temp dir should be created");
File diff suppressed because it is too large Load Diff
+3 -70
View File
@@ -444,29 +444,12 @@ async fn prepare_rename_commit_and_destination_metadata(
#[derive(Debug)]
pub(in crate::disk) struct LocalRenamePreflightRejection(());
impl LocalRenamePreflightRejection {
pub(in crate::disk) fn not_dispatched() -> Self {
Self(())
}
}
#[derive(Default)]
pub(super) struct RenameDataState {
namespace_owner: Option<Arc<dyn Send + Sync>>,
metadata_reservation: Option<os::MetadataMutationReservation>,
preflight_rejection: Option<LocalRenamePreflightRejection>,
}
#[cfg(test)]
impl RenameDataState {
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
Self {
metadata_reservation: Some(reservation),
..Default::default()
}
}
}
// Keep optional native MPU state machines out of ordinary rename frames.
// Construct the future here as well as pinning it, so its temporary stays off
// the caller's poll stack. Call only from the existing native MPU branches.
@@ -508,15 +491,6 @@ impl LocalDisk {
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
}
let mut fi = fi;
// Pre-acquired metadata must never reverse quota or multipart-source
// lock order. These paths need their own reservation protocol.
if state.metadata_reservation.is_some()
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
{
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
@@ -612,27 +586,7 @@ impl LocalDisk {
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
None => state.namespace_owner.take(),
};
// A reserved lease is single-use and bound to the exact disk path.
// Keep the independent namespace/source owner alongside it; a pre-held
// mutex cannot be acquired again without deadlocking.
let reservation_fence = state
.metadata_reservation
.as_ref()
.and_then(|reservation| reservation.fence());
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
if multipart_staging.is_some() || quota_fence_claim.is_some() {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
let lease = reservation
.into_lease_for(&root_destination_object_path, self.reservation_instance)
.inspect_err(|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
})?;
Arc::new((lease, metadata_owner))
} else {
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
};
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
drop(metadata_wait);
#[cfg(unix)]
let parent_receipt = if native_timing {
@@ -646,23 +600,17 @@ impl LocalDisk {
};
#[cfg(unix)]
let mut receipt_mutation = native_timing
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
&self.root,
dst_volume,
&root_destination_object_path,
staging_prefix.as_deref(),
Some(metadata_owner),
Some(metadata_lease),
)
.await;
drop(namespace_wait);
// The volume read lease prevents deletion of this disk's bucket until
// publication finishes. Recheck after its wait, before any mutation.
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
return Err(DiskError::DiskOngoingReq);
}
if let Some(claim) = quota_fence_claim {
mutation_lease.attach_external_guard(claim);
}
@@ -2601,23 +2549,8 @@ impl LocalDisk {
dst_path: &str,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> super::super::RenameDataObservation {
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
.await
}
pub(in crate::disk) async fn rename_data_observed_with_reservation(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
) -> super::super::RenameDataObservation {
let (namespace_owner, metadata_reservation) = owners;
let mut state = RenameDataState {
namespace_owner,
metadata_reservation,
..Default::default()
};
let result = self
+12 -112
View File
@@ -106,8 +106,6 @@ pub(crate) struct RenameDataGuards {
pub(crate) scanner_publication_lease_token: Option<Uuid>,
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
pub(crate) metadata_reservation_token: Option<Uuid>,
}
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
@@ -124,13 +122,6 @@ impl RenameDataObservation {
}
}
fn not_dispatched(error: DiskError) -> Self {
Self {
result: Err(error),
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
}
}
pub(crate) fn rejected_before_publication(&self) -> bool {
self.result.is_err() && self.preflight_rejection.is_some()
}
@@ -320,82 +311,6 @@ pub enum Disk {
}
impl Disk {
/// Prepare an inert single-use reservation on this disk.
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
pub async fn prepare_object_metadata(
&self,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> Result<Uuid> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.prepare_object_metadata(bucket, object),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
}
}
/// Refresh an inert token while its sender stages the write.
pub async fn renew_object_metadata(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: std::time::Duration,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.renew_object_metadata(bucket, object, token),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await,
}
}
/// Enqueue a prepared token in object mutation order.
pub async fn activate_object_metadata(
&self,
bucket: &str,
object: &str,
token: Uuid,
timeout: std::time::Duration,
) -> Result<()> {
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
match self {
Disk::Local(disk) => tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
disk.activate_object_metadata(bucket, object, token),
)
.await
.map_err(|_| DiskError::Timeout)?,
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
}
}
/// Cancel an unclaimed reservation. False does not prove that a claim has
/// not already started a physical mutation.
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
match self {
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
}
}
/// Physical ownership belongs to the node hosting the local attachment.
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
match self {
@@ -1336,33 +1251,18 @@ impl Disk {
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
.await
}
Disk::Remote(remote_disk) => {
if guards.metadata_reservation.is_some() {
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
}
if let Some(token) = guards.metadata_reservation_token {
if guards.scanner_publication_lease_token.is_some() {
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
}
return RenameDataObservation::unknown(
remote_disk
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
.await,
);
}
RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
guards.scanner_publication_lease_token,
)
.await,
)
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
guards.scanner_publication_lease_token,
)
.await,
),
}
}
-349
View File
@@ -22,10 +22,8 @@ use parking_lot::Mutex;
use rustfs_utils::path::SLASH_SEPARATOR;
use std::{
collections::{HashMap, VecDeque},
future::Future,
io,
path::{Component, Path, PathBuf},
pin::Pin,
sync::{Arc, LazyLock, Weak},
time::{Duration, Instant},
};
@@ -2231,353 +2229,6 @@ pub(crate) struct MetadataMutationLease {
_owner: Option<Arc<dyn Send + Sync>>,
}
pub(crate) trait MetadataReservationFence: Send + Sync {
fn is_valid(&self) -> bool;
}
/// Single-use ownership of the metadata lock for an exact physical object.
pub(crate) struct MetadataMutationReservation {
key: PathBuf,
disk_instance: uuid::Uuid,
lease: MetadataReservationLease,
capacity: Option<OwnedSemaphorePermit>,
fence: Option<Arc<dyn MetadataReservationFence>>,
}
enum MetadataReservationLease {
Held(Arc<MetadataMutationLease>),
Queued(Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>>),
}
impl MetadataMutationReservation {
pub(crate) fn fence(&self) -> Option<Arc<dyn MetadataReservationFence>> {
self.fence.clone()
}
/// Wait for a previously enqueued lease without holding an I/O permit.
/// Dropping this future removes an unclaimed waiter from the FIFO queue.
pub(crate) async fn wait(self) -> Self {
let Self {
key,
disk_instance,
lease,
capacity,
fence,
} = self;
let lease = match lease {
MetadataReservationLease::Held(lease) => lease,
MetadataReservationLease::Queued(waiter) => waiter.await,
};
Self {
key,
disk_instance,
lease: MetadataReservationLease::Held(lease),
capacity,
fence,
}
}
pub(crate) fn into_lease_for(self, key: &Path, disk_instance: uuid::Uuid) -> Result<Arc<dyn Send + Sync>> {
if self.key != key || self.disk_instance != disk_instance {
return Err(DiskError::InvalidPath);
}
if self.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
return Err(DiskError::DiskOngoingReq);
}
let MetadataReservationLease::Held(lease) = self.lease else {
return Err(DiskError::DiskOngoingReq);
};
Ok(match self.capacity {
Some(capacity) => Arc::new((lease, capacity)),
None => lease,
})
}
}
pub(crate) async fn reserve_metadata_mutation_lease(
object: &Path,
disk_instance: uuid::Uuid,
owner: Option<Arc<dyn Send + Sync>>,
) -> MetadataMutationReservation {
MetadataMutationReservation {
key: object.to_path_buf(),
disk_instance,
lease: MetadataReservationLease::Held(acquire_metadata_mutation_lease(object, owner).await),
capacity: None,
fence: None,
}
}
struct MetadataReservationEntry {
state: MetadataReservationState,
expires_at: tokio::time::Instant,
}
enum MetadataReservationState {
Prepared {
key: PathBuf,
disk_instance: uuid::Uuid,
capacity: Option<OwnedSemaphorePermit>,
owner: Option<Arc<dyn MetadataReservationFence>>,
},
Queued(MetadataMutationReservation),
}
/// Process-local, bounded one-shot reservations for one disk instance.
pub(crate) struct MetadataReservationRegistry {
entries: Mutex<HashMap<uuid::Uuid, MetadataReservationEntry>>,
capacity: Arc<Semaphore>,
}
impl MetadataReservationRegistry {
pub(crate) fn new(limit: usize) -> Arc<Self> {
let registry = Arc::new(Self {
entries: Mutex::new(HashMap::new()),
capacity: Arc::new(Semaphore::new(limit)),
});
let weak = Arc::downgrade(&registry);
tokio::spawn(async move {
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let Some(registry) = weak.upgrade() else { break };
registry.remove_expired();
}
});
registry
}
pub(crate) async fn reserve(
&self,
key: &Path,
disk_instance: uuid::Uuid,
acquire_timeout: Duration,
owner: Option<Arc<dyn Send + Sync>>,
) -> Result<uuid::Uuid> {
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
Ok(capacity) => capacity,
Err(_) => {
self.remove_expired();
Arc::clone(&self.capacity)
.try_acquire_owned()
.map_err(|_| DiskError::DiskOngoingReq)?
}
};
let mut reservation = tokio::time::timeout(acquire_timeout, reserve_metadata_mutation_lease(key, disk_instance, owner))
.await
.map_err(|_| DiskError::Timeout)?;
reservation.capacity = Some(capacity);
let token = uuid::Uuid::new_v4();
let mut entries = self.entries.lock();
match entries.entry(token) {
std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry {
state: MetadataReservationState::Queued(reservation),
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
}),
std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected),
};
Ok(token)
}
/// Register this mutation in FIFO order before returning its token.
/// The mutex may still be held by an earlier operation when this returns.
pub(crate) async fn enqueue(&self, key: &Path, disk_instance: uuid::Uuid) -> Result<uuid::Uuid> {
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
Ok(capacity) => capacity,
Err(_) => {
self.remove_expired();
Arc::clone(&self.capacity)
.try_acquire_owned()
.map_err(|_| DiskError::DiskOngoingReq)?
}
};
let object = key.to_path_buf();
let mut waiter: Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>> =
Box::pin(async move { acquire_metadata_mutation_lease(&object, None).await });
// Poll once so Tokio registers this waiter before the token can be
// acknowledged; retaining the future retains its position in the queue.
let lease = match futures::poll!(waiter.as_mut()) {
std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease),
std::task::Poll::Pending => MetadataReservationLease::Queued(waiter),
};
let token = uuid::Uuid::new_v4();
let reservation = MetadataMutationReservation {
key: key.to_path_buf(),
disk_instance,
lease,
capacity: Some(capacity),
fence: None,
};
let mut entries = self.entries.lock();
match entries.entry(token) {
std::collections::hash_map::Entry::Vacant(entry) => entry.insert(MetadataReservationEntry {
state: MetadataReservationState::Queued(reservation),
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
}),
std::collections::hash_map::Entry::Occupied(_) => return Err(DiskError::Unexpected),
};
Ok(token)
}
/// Allocate an inert token before a client asks to enter the FIFO queue.
/// A lost preparation response cannot hold an object metadata lock.
#[cfg(test)]
pub(crate) fn prepare(&self, key: &Path, disk_instance: uuid::Uuid) -> Result<uuid::Uuid> {
self.prepare_with_owner(key, disk_instance, None)
}
pub(crate) fn prepare_with_owner(
&self,
key: &Path,
disk_instance: uuid::Uuid,
owner: Option<Arc<dyn MetadataReservationFence>>,
) -> Result<uuid::Uuid> {
let capacity = match Arc::clone(&self.capacity).try_acquire_owned() {
Ok(capacity) => capacity,
Err(_) => {
self.remove_expired();
Arc::clone(&self.capacity)
.try_acquire_owned()
.map_err(|_| DiskError::DiskOngoingReq)?
}
};
let token = uuid::Uuid::new_v4();
let entry = MetadataReservationEntry {
state: MetadataReservationState::Prepared {
key: key.to_path_buf(),
disk_instance,
capacity: Some(capacity),
owner,
},
expires_at: tokio::time::Instant::now() + Duration::from_secs(60),
};
match self.entries.lock().entry(token) {
std::collections::hash_map::Entry::Vacant(slot) => {
slot.insert(entry);
Ok(token)
}
std::collections::hash_map::Entry::Occupied(_) => Err(DiskError::Unexpected),
}
}
/// Extend an inert token without reviving an expired or cancelled entry.
pub(crate) fn renew(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<()> {
let mut entries = self.entries.lock();
let entry = entries.get_mut(&token).ok_or(DiskError::DiskOngoingReq)?;
let now = tokio::time::Instant::now();
if entry.expires_at <= now {
return Err(DiskError::Timeout);
}
let MetadataReservationState::Prepared {
key: stored_key,
disk_instance: stored_instance,
owner,
..
} = &entry.state
else {
return Err(DiskError::DiskOngoingReq);
};
if stored_key != key || *stored_instance != disk_instance {
return Err(DiskError::InvalidPath);
}
if owner.as_ref().is_some_and(|fence| !fence.is_valid()) {
return Err(DiskError::DiskOngoingReq);
}
entry.expires_at = now + Duration::from_secs(60);
Ok(())
}
/// Atomically enqueue a prepared token. Cancellation and enqueue serialize
/// on the registry, so a delayed enqueue cannot revive a cancelled token.
pub(crate) async fn activate(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<()> {
std::future::poll_fn(|cx| {
let mut entries = self.entries.lock();
let Some(entry) = entries.get_mut(&token) else {
return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq));
};
if entry.expires_at <= tokio::time::Instant::now() {
return std::task::Poll::Ready(Err(DiskError::Timeout));
}
let (stored_key, stored_instance, capacity, owner) = match &mut entry.state {
MetadataReservationState::Prepared {
key,
disk_instance,
capacity,
owner,
} => (key, disk_instance, capacity, owner),
MetadataReservationState::Queued(reservation) => {
return std::task::Poll::Ready(if reservation.key != key || reservation.disk_instance != disk_instance {
Err(DiskError::InvalidPath)
} else if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
Err(DiskError::DiskOngoingReq)
} else {
Ok(())
});
}
};
if stored_key != key || *stored_instance != disk_instance {
return std::task::Poll::Ready(Err(DiskError::InvalidPath));
}
if owner.as_ref().is_some_and(|fence| !fence.is_valid()) {
return std::task::Poll::Ready(Err(DiskError::DiskOngoingReq));
}
let Some(capacity) = capacity.take() else {
return std::task::Poll::Ready(Err(DiskError::Unexpected));
};
let object = stored_key.clone();
let fence = owner.take();
let owner: Option<Arc<dyn Send + Sync>> = fence.clone().map(|fence| fence as Arc<dyn Send + Sync>);
let mut waiter: Pin<Box<dyn Future<Output = Arc<MetadataMutationLease>> + Send>> =
Box::pin(async move { acquire_metadata_mutation_lease(&object, owner).await });
let lease = match waiter.as_mut().poll(cx) {
std::task::Poll::Ready(lease) => MetadataReservationLease::Held(lease),
std::task::Poll::Pending => MetadataReservationLease::Queued(waiter),
};
entry.state = MetadataReservationState::Queued(MetadataMutationReservation {
key: stored_key.clone(),
disk_instance,
lease,
capacity: Some(capacity),
fence,
});
std::task::Poll::Ready(Ok(()))
})
.await
}
pub(crate) fn claim(&self, token: uuid::Uuid, key: &Path, disk_instance: uuid::Uuid) -> Result<MetadataMutationReservation> {
let entry = self.entries.lock().remove(&token).ok_or(DiskError::DiskOngoingReq)?;
if entry.expires_at <= tokio::time::Instant::now() {
return Err(DiskError::Timeout);
}
let MetadataReservationState::Queued(reservation) = entry.state else {
return Err(DiskError::DiskOngoingReq);
};
if reservation.key != key || reservation.disk_instance != disk_instance {
return Err(DiskError::InvalidPath);
}
if reservation.fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
return Err(DiskError::DiskOngoingReq);
}
Ok(reservation)
}
pub(crate) fn cancel(&self, token: uuid::Uuid) -> bool {
let removed = self.entries.lock().remove(&token);
removed.is_some()
}
fn remove_expired(&self) {
let now = tokio::time::Instant::now();
let expired: Vec<_> = self
.entries
.lock()
.extract_if(|_, entry| entry.expires_at <= now)
.map(|(_, entry)| entry)
.collect();
drop(expired);
}
}
pub(crate) async fn acquire_metadata_mutation_lease(
object: &Path,
owner: Option<Arc<dyn Send + Sync>>,
+1 -38
View File
@@ -28,7 +28,6 @@ use tokio_util::sync::CancellationToken;
#[derive(Clone)]
pub struct NamespaceLockFence {
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
local_probes: Arc<Vec<rustfs_lock::fast_lock::guard::FastLockGuardProbe>>,
#[cfg(test)]
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
}
@@ -45,7 +44,6 @@ impl NamespaceLockFence {
pub(crate) fn new() -> Self {
Self {
signals: Arc::default(),
local_probes: Arc::default(),
#[cfg(test)]
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
}
@@ -60,28 +58,17 @@ impl NamespaceLockFence {
{
return true;
}
self.signals.iter().any(|signal| signal.is_lost()) || self.local_probes.iter().any(|probe| probe.is_lost())
self.signals.iter().any(|signal| signal.is_lost())
}
pub(crate) fn add_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
Arc::make_mut(&mut self.signals).push(signal);
}
fn add_local_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
if let rustfs_lock::NamespaceLockGuard::Fast(guard) = guard
&& let Some(probe) = guard.release_probe()
{
Arc::make_mut(&mut self.local_probes).push(probe);
}
}
fn extend(&mut self, other: &Self) {
if !Arc::ptr_eq(&self.signals, &other.signals) {
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
}
if !Arc::ptr_eq(&self.local_probes, &other.local_probes) {
Arc::make_mut(&mut self.local_probes).extend(other.local_probes.iter().cloned());
}
#[cfg(test)]
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
@@ -119,7 +106,6 @@ impl NamespaceLockSignalTestFence {
) -> Self {
let fence = NamespaceLockFence {
signals: Arc::default(),
local_probes: Arc::default(),
forced_lost: Arc::new(vec![loss_handle]),
};
let signal_key = Arc::as_ptr(signal) as usize;
@@ -174,20 +160,6 @@ pub struct ObjectLockConfigSnapshot {
}
impl ObjectLockConfigSnapshot {
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
self._lifecycle_guard.is_some()
}
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
self.bucket.as_deref() == Some(bucket)
&& incarnation.is_some()
&& self.bucket_incarnation_id == incarnation
&& self
._lifecycle_guard
.as_ref()
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
}
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
Self {
store_id: None,
@@ -236,7 +208,6 @@ impl ObjectLockConfigSnapshot {
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
lifecycle_fence.add_signal(signal);
}
lifecycle_fence.add_local_guard(&lifecycle_guard);
Self {
store_id: Some(store_id),
bucket: Some(bucket.to_string()),
@@ -1382,13 +1353,6 @@ impl ObjectOptions {
if let Some(signal) = guard.lock_lost_signal() {
self.add_namespace_lock_lost_signal(signal);
}
if let rustfs_lock::NamespaceLockGuard::Fast(local) = guard
&& !local.is_disabled()
{
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.add_local_guard(guard);
}
}
pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
@@ -1396,7 +1360,6 @@ impl ObjectOptions {
if let Some(signal) = guard.lock_lost_signal() {
fence.add_signal(signal);
}
fence.add_local_guard(guard);
}
pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
-8
View File
@@ -101,12 +101,6 @@ pub(crate) struct NamespaceCommitGuard {
counted: bool,
}
impl NamespaceCommitGuard {
pub(crate) fn covers_instance(&self, ctx: &Arc<InstanceContext>) -> bool {
Arc::ptr_eq(&self.ctx, ctx)
}
}
impl Drop for NamespaceCommitGuard {
fn drop(&mut self) {
if self.counted {
@@ -1157,8 +1151,6 @@ mod tests {
other.set_scanner_publication_state(false);
assert!(first.scanner_publication_state_allowed());
let one = first.begin_namespace_commit();
assert!(one.covers_instance(&first));
assert!(!one.covers_instance(&other));
let shared_owner = Arc::clone(&one);
let two = first.begin_namespace_commit();
assert!(first.namespace_commits_pending());
@@ -2538,7 +2538,6 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
pub(in crate::set_disk) committed_file_info: FileInfo,
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
}
pub(in crate::set_disk) struct RenameTailCleanup {
@@ -3469,8 +3468,6 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
rollback_receipt: Option<RenameRollbackReceipt>,
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
}
impl<'a> RenameDataFenceOptions<'a> {
@@ -3486,8 +3483,6 @@ impl<'a> RenameDataFenceOptions<'a> {
rollback_receipt: None,
multipart_proof_authority: None,
multipart_commit_owner: None,
reservation_ctx: None,
prepared_reservations: None,
}
}
@@ -3524,274 +3519,6 @@ impl<'a> RenameDataFenceOptions<'a> {
self.namespace_commit_guard = namespace_commit_guard;
self
}
pub(in crate::set_disk) fn with_reservation_context(
mut self,
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
) -> Self {
self.reservation_ctx = ctx;
self.prepared_reservations = reservations;
self
}
}
#[derive(Clone)]
struct MetadataRenewalTarget {
disk: DiskStore,
token: Uuid,
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
}
struct MetadataRenewalGroup {
targets: Arc<parking_lot::Mutex<HashMap<usize, MetadataRenewalTarget>>>,
task: tokio::task::JoinHandle<()>,
}
impl Drop for MetadataRenewalGroup {
fn drop(&mut self) {
self.task.abort();
}
}
pub(crate) struct RenameMetadataReservation {
disk: DiskStore,
token: Option<Uuid>,
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
renewal: Option<(Arc<MetadataRenewalGroup>, usize)>,
}
impl RenameMetadataReservation {
fn stop_renewal(&mut self) {
if let Some((group, index)) = self.renewal.take() {
let mut targets = group.targets.lock();
targets.remove(&index);
if targets.is_empty() {
group.task.abort();
}
}
}
async fn cancel(&mut self) -> disk::error::Result<()> {
self.stop_renewal();
let Some(token) = self.token else { return Ok(()) };
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
Ok(Ok(true)) => {
self.token = None;
Ok(())
}
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
Ok(Err(error)) => Err(error),
Err(_) => Err(DiskError::Timeout),
}
}
}
impl Drop for RenameMetadataReservation {
fn drop(&mut self) {
self.stop_renewal();
let Some(token) = self.token.take() else { return };
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
let disk = self.disk.clone();
runtime.spawn(async move {
let _ = disk.cancel_object_metadata_reservation(token).await;
});
}
}
}
pub(crate) async fn prepare_rename_metadata_reservations(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
incarnation: Uuid,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
owns_bucket_fence: bool,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
let prepared: Vec<disk::error::Result<Option<RenameMetadataReservation>>> = join_all(disks.iter().map(|disk| async move {
let Some(disk) = disk else { return Ok(None) };
if disk.is_local() {
crate::store::admit_local_disk(ctx, disk, None, false).await?;
}
let prepared_at = tokio::time::Instant::now();
let token = if disk.is_local() && owns_bucket_fence {
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
.await
.ok_or(DiskError::DiskOngoingReq)?;
store
.reserve_local_object_metadata_at_incarnation(
&disk.endpoint().to_string(),
bucket,
object,
incarnation,
Duration::from_secs(5),
)
.await?
} else {
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
.await?
};
// No await separates receipt from installing the cancellation owner.
Ok(Some(RenameMetadataReservation {
disk: disk.clone(),
token: Some(token),
renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)),
renewal: None,
}))
}))
.await;
let mut reservations = Vec::with_capacity(prepared.len());
let mut failed = false;
for result in prepared {
match result {
Ok(reservation) => reservations.push(reservation),
Err(_) => {
failed = true;
reservations.push(None);
}
}
}
if failed {
// Prepared tokens are inert; dropping them cannot leave a queued writer.
return Ok(None);
}
let targets = Arc::new(parking_lot::Mutex::new(HashMap::new()));
for (index, reservation) in reservations.iter().enumerate() {
if let Some(reservation) = reservation {
let token = reservation.token.ok_or(DiskError::Unexpected)?;
targets.lock().insert(
index,
MetadataRenewalTarget {
disk: reservation.disk.clone(),
token,
renewed_at: Arc::clone(&reservation.renewed_at),
},
);
}
}
if !targets.lock().is_empty() {
let renewal_targets = Arc::clone(&targets);
let renewal_ctx = Arc::clone(ctx);
let renewal_bucket = bucket.to_owned();
let renewal_object = object.to_owned();
// One task renews the whole fanout; short writes create no renewal RPCs.
let task = tokio::spawn(async move {
let mut delay = Duration::from_secs(20);
loop {
tokio::time::sleep(delay).await;
let current: Vec<_> = renewal_targets.lock().values().cloned().collect();
if current.is_empty() {
break;
}
// Once any acknowledgment is too old, the complete fanout can
// no longer be safely activated. Stop retrying stale peers;
// their receiver-side tokens still expire on their own.
if current
.iter()
.any(|target| target.renewed_at.lock().elapsed() >= Duration::from_secs(60))
{
break;
}
let results = join_all(current.into_iter().map(|target| {
let ctx = Arc::clone(&renewal_ctx);
let bucket = &renewal_bucket;
let object = &renewal_object;
async move {
let started = tokio::time::Instant::now();
let admitted = !target.disk.is_local()
|| crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok();
if admitted
&& target
.disk
.renew_object_metadata(bucket, object, target.token, Duration::from_secs(5))
.await
.is_ok()
{
*target.renewed_at.lock() = started;
true
} else {
false
}
}
}))
.await;
delay = if results.into_iter().all(|success| success) {
Duration::from_secs(20)
} else {
Duration::from_secs(5)
};
}
});
let group = Arc::new(MetadataRenewalGroup { targets, task });
for (index, reservation) in reservations.iter_mut().enumerate() {
if let Some(reservation) = reservation {
reservation.renewal = Some((Arc::clone(&group), index));
}
}
}
Ok(Some(reservations))
}
pub(crate) fn align_rename_metadata_reservations(
mut reservations: Vec<Option<RenameMetadataReservation>>,
disks: &[Option<DiskStore>],
) -> Option<Vec<Option<RenameMetadataReservation>>> {
if reservations.len() != disks.len() {
return None;
}
let mut aligned = Vec::with_capacity(disks.len());
for disk in disks {
let Some(disk) = disk else {
aligned.push(None);
continue;
};
let index = reservations
.iter()
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
aligned.push(reservations[index].take());
}
// Disks lost while staging need no activation; dropping their still-inert
// reservations schedules cancellation without blocking the commit.
Some(aligned)
}
async fn activate_rename_metadata_reservations(
mut reservations: Vec<Option<RenameMetadataReservation>>,
bucket: &str,
object: &str,
ctx: &Arc<crate::runtime::instance::InstanceContext>,
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
// Freeze renewal before activation. A failed refresh keeps the last
// acknowledged timestamp, so stale tokens fall back before dispatch.
for reservation in reservations.iter_mut().flatten() {
reservation.stop_renewal();
}
if reservations
.iter()
.flatten()
.any(|reservation| reservation.renewed_at.lock().elapsed() >= Duration::from_secs(50))
{
return Ok(None);
}
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
if reservation.disk.is_local() {
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
}
let token = reservation.token.ok_or(DiskError::Unexpected)?;
reservation
.disk
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
.await
}))
.await;
if activated.into_iter().any(|result| result.is_err()) {
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
if cancelled.into_iter().any(|result| result.is_err()) {
return Err(DiskError::DiskOngoingReq);
}
return Ok(None);
}
Ok(Some(reservations))
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3803,7 +3530,7 @@ type RenameDataLegacyTuple = (
Option<OldCurrentSize>,
);
pub(crate) fn put_rename_early_ack_enabled() -> bool {
fn put_rename_early_ack_enabled() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
}
@@ -4169,7 +3896,6 @@ impl SetDisks {
old_current_source,
committed_file_info,
tail_drain: None,
tail_has_metadata_reservations: false,
})
}
@@ -4266,8 +3992,6 @@ impl SetDisks {
rollback_receipt,
multipart_proof_authority: _,
multipart_commit_owner: _,
reservation_ctx,
prepared_reservations,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4283,11 +4007,6 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
(_, _) => None,
};
let tail_has_metadata_reservations = reservations.is_some();
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
@@ -4320,11 +4039,6 @@ impl SetDisks {
let file_info = file_info.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let namespace_commit_guard = namespace_commit_guard.clone();
let reservation = reservations
.as_mut()
.and_then(|entries| entries.get_mut(i))
.and_then(Option::take);
let reservation_ctx = reservation_ctx.clone();
tasks.spawn(async move {
let _namespace_commit_guard = namespace_commit_guard.clone();
let mut dispatch_state = RenameDispatchState::NotDispatched;
@@ -4335,17 +4049,6 @@ impl SetDisks {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let disk_admission = if reservation.is_some() && disk.is_local() {
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
// The fanout owner already covers this instance through
// physical completion; only add a disk owner when it does not.
let covered = namespace_commit_guard
.as_ref()
.is_some_and(|guard| guard.covers_instance(ctx));
crate::store::admit_local_disk(ctx, &disk, None, !covered).await?
} else {
None
};
let disk_wait_location_stage = if disk.is_local() {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
} else {
@@ -4371,11 +4074,6 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
dispatch_state = RenameDispatchState::MayHavePublished;
let _foreground_active_guard = rename_foreground_active_guard(i);
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
} else {
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
};
let observed = disk
.rename_data_borrowed_with_fence_observed(
&src_bucket,
@@ -4385,8 +4083,9 @@ impl SetDisks {
&dst_object,
disk::RenameDataGuards {
scanner_publication_lease_token,
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
namespace_owner,
namespace_owner: namespace_commit_guard
.clone()
.map(|owner| owner as Arc<dyn Send + Sync>),
..Default::default()
},
)
@@ -4664,7 +4363,6 @@ impl SetDisks {
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
commit.map(|mut commit| {
commit.tail_drain = Some(tail_drain);
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
attach_tail_owner_started,
@@ -4734,8 +4432,6 @@ impl SetDisks {
rollback_receipt,
multipart_proof_authority,
multipart_commit_owner,
reservation_ctx: _,
prepared_reservations: _,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -5205,7 +4901,6 @@ impl SetDisks {
old_current_source,
committed_file_info,
tail_drain: None,
tail_has_metadata_reservations: false,
})
}
@@ -7924,45 +7619,6 @@ mod tests {
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() {
let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await;
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let endpoint = disk.endpoint().to_string();
ctx.local_disk_map()
.write()
.await
.insert(endpoint.clone(), Some(disk.clone()));
let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false)
.await
.expect("prepare")
.expect("complete fanout");
let group = Arc::clone(
&reservations[0]
.as_ref()
.expect("reservation")
.renewal
.as_ref()
.expect("renewal")
.0,
);
ctx.local_disk_map().write().await.remove(&endpoint);
tokio::task::yield_now().await;
tokio::time::pause();
tokio::time::advance(Duration::from_secs(21)).await;
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(41)).await;
for _ in 0..10 {
if group.task.is_finished() {
break;
}
tokio::task::yield_now().await;
}
tokio::time::resume();
assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts");
drop(reservations);
}
#[tokio::test]
async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() {
use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint};
+7 -13
View File
@@ -410,10 +410,6 @@ impl ObjectLockDiagGuard {
self
}
async fn downgrade_committed_tail(&mut self) -> bool {
self.guard.downgrade_committed_tail().await
}
/// Whether the underlying namespace lock's heartbeat has observed a
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
/// point on this so a stale lock holder does not race a double-write.
@@ -906,10 +902,6 @@ mod core;
pub(crate) use core::io_primitives::disk_call_counters;
#[cfg(test)]
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
pub(crate) use core::io_primitives::{
RenameMetadataReservation, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
put_rename_early_ack_enabled,
};
mod ctx;
mod metadata;
mod ops;
@@ -9046,6 +9038,13 @@ mod tests {
"constructing an object lock must not clone each client Arc"
);
// Check shared ownership before acquisition can spawn pending cleanup.
let read_lock = set
.new_ns_lock("bucket", "read-object")
.await
.expect("second namespace lock should be created");
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
let write_error = write_lock
.get_write_lock(Duration::from_millis(500))
.await
@@ -9060,11 +9059,6 @@ mod tests {
),
"the shared client representation must preserve the exact write quorum result: {write_error}"
);
let read_lock = set
.new_ns_lock("bucket", "read-object")
.await
.expect("second namespace lock should be created");
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
let read_guard = read_lock
.get_read_lock(Duration::from_millis(500))
.await
+9 -493
View File
@@ -34,7 +34,6 @@ mod heal_lock_wait_tests;
#[derive(Clone, Copy)]
struct ExplicitVersionHeal<'a> {
opts: &'a HealOpts,
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
allow_regeneration: bool,
preserve_unrepairable: bool,
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
@@ -42,58 +41,6 @@ struct ExplicitVersionHeal<'a> {
retirement: Option<&'a MarkerRetirementContext<'a>>,
}
async fn restore_heal_exclusive_guard(
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
) -> disk::error::Result<()> {
tokio::time::timeout(get_lock_acquire_timeout(), async {
loop {
{
let mut guard = guard.lock().await;
if guard.is_lock_lost() {
return Err(DiskError::DiskOngoingReq);
}
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
if !matches!(waiting, Ok(Some(false))) {
return Err(DiskError::Timeout);
}
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
.await
.unwrap_or(false)
{
// A writer may have waited on a peer that upgraded while
// another peer was still waiting for its readers to drain.
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
return if matches!(waiting, Ok(Some(false))) {
Ok(())
} else {
Err(DiskError::Timeout)
};
}
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.map_err(|_| DiskError::Timeout)?
}
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
loop {
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let waiting =
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
if !matches!(waiting, Ok(Some(false))) {
return;
}
}
}
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
drop(tokio::spawn(async move {
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
}));
}
// Keep the large heal state machine and its construction temporary out of
// each dispatch caller's poll frame. Ordinary object writes do not use this
// heal-only allocation boundary.
@@ -897,7 +844,6 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts,
staging_guard: None,
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: None,
@@ -1048,7 +994,6 @@ impl SetDisks {
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
let ExplicitVersionHeal {
opts,
staging_guard,
allow_regeneration: allow_explicit_version_regen,
preserve_unrepairable,
commit_fence,
@@ -1756,29 +1701,7 @@ impl SetDisks {
let dst_data_dir = data_dir;
if !latest_meta.deleted && !latest_meta.is_remote() {
let mut shared_staging = false;
if let Some(guard) = staging_guard {
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
guard.lock().await.supports_heal_lock_transition().await
})
.await
.unwrap_or(false);
if supported {
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
guard.lock().await.downgrade_committed_tail().await
})
.await
.unwrap_or(false);
if converted {
shared_staging = true;
} else {
// Some peers may have converted. Abort without committing;
// dropping the guard releases each retained lease.
return Err(DiskError::Timeout);
}
}
}
let stage = async {
let stage_result: disk::error::Result<Option<DiskError>> = async {
let erasure_info = latest_meta.erasure.clone();
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
@@ -2003,43 +1926,17 @@ impl SetDisks {
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
Ok(None)
};
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
if let Some(guard) = staging_guard {
tokio::select! {
result = stage => result,
_ = wait_for_heal_writer(guard) => {
// The UUID prefix is private to this attempt. Cleanup must not
// retain the namespace lock while a foreground writer waits;
// the stale-tmp sweeper also covers a cancelled disk operation.
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
return Err(DiskError::Timeout);
}
}
} else {
stage.await
}
} else {
stage.await
};
}
.await;
match stage_result {
Ok(None) => {}
failure => {
if shared_staging {
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
} else {
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
}
// Release readers and writers before removing all parts
// staged under this heal's exclusively owned prefix.
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
return failure.map(|error| (result, error));
}
}
if shared_staging
&& let Some(guard) = staging_guard
&& let Err(error) = restore_heal_exclusive_guard(guard).await
{
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
return Err(error);
}
}
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
if _read_repair_read_lock_guard
@@ -3719,7 +3616,6 @@ impl SetDisks {
&version,
ExplicitVersionHeal {
opts: &opts,
staging_guard: None,
allow_regeneration: false,
preserve_unrepairable: true,
commit_fence: Some(can_commit),
@@ -3780,7 +3676,6 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &read_opts,
staging_guard: None,
allow_regeneration: false,
preserve_unrepairable: true,
commit_fence: Some(&can_commit),
@@ -3807,7 +3702,7 @@ impl SetDisks {
.new_ns_lock(bucket, object)
.await
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
Some(Arc::new(tokio::sync::Mutex::new(
Some(Arc::new(
measure_heal_object_lock(
"heal_object",
HealObjectLockKind::Write,
@@ -3819,7 +3714,7 @@ impl SetDisks {
.narrow_to_disk()
.unwrap_or_else(DiskError::other)
})?,
)))
))
} else {
None
};
@@ -3837,18 +3732,10 @@ impl SetDisks {
} else {
None
};
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
guard.lock().await.lock_lost_signal()
} else {
None
};
let locks_lost = || {
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
};
// Recovery cleanup can outlive its future. Never permit reads until
// every physical operation holding its recovery owner has finished.
let recovery_tail = Arc::new(());
let mut recovery_removed_version = false;
if !opts.read_repair
&& _write_lock_guard.is_some()
@@ -3875,7 +3762,6 @@ impl SetDisks {
Arc::new((
scope.clone(),
self.ctx.begin_namespace_commit(),
Arc::clone(&recovery_tail),
_write_lock_guard.clone(),
transaction_guard.clone(),
)) as Arc<dyn Send + Sync>
@@ -3999,9 +3885,6 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &inner_opts,
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
.then_some(_write_lock_guard.as_ref())
.flatten(),
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&commit_fence),
@@ -4026,9 +3909,6 @@ impl SetDisks {
version_id,
ExplicitVersionHeal {
opts: &inner_opts,
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
.then_some(_write_lock_guard.as_ref())
.flatten(),
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&commit_fence),
@@ -6167,7 +6047,6 @@ mod heal_result_report_tests {
"",
ExplicitVersionHeal {
opts: &opts,
staging_guard: None,
allow_regeneration: true,
preserve_unrepairable: false,
commit_fence: Some(&can_commit),
@@ -10758,369 +10637,6 @@ mod heal_result_report_tests {
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-storage-entry";
let object = "object.bin";
let expected = vec![0x37; 1024 * 1024];
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(expected.clone()),
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write object");
let metadata = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external object data").to_string())
.join("part.1");
tokio::fs::remove_file(&missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal = tokio::spawn(async move {
let mut absence = None;
let result = heal_set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
read_repair: false,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await;
(result, absence)
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("read-repair reached shard reconstruction");
let mut reader = tokio::time::timeout(
Duration::from_millis(200),
set.get_object_reader(
bucket,
object,
None,
http::HeaderMap::new(),
&ObjectOptions {
suppress_read_repair: true,
..Default::default()
},
),
)
.await
.expect("background heal reconstruction should allow concurrent readers")
.expect("open object while reconstruction is paused");
let mut during_heal = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
.await
.expect("read completes while reconstruction is paused")
.expect("read object during reconstruction");
assert_eq!(during_heal, expected);
drop(reader);
release.notify_one();
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("repair finishes")
.expect("repair task joins");
let (_, error) = result.expect("repair operation succeeds");
assert!(error.is_none(), "repair failed: {error:?}");
assert!(absence.is_none());
assert!(missing.exists(), "missing shard was not restored");
let mut reader = set
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read repaired object");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read full object");
assert_eq!(actual, expected);
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-long-put";
let object = "object.bin";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original object");
let metadata = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external data").to_string())
.join("part.1");
tokio::fs::remove_file(missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal = tokio::spawn(async move {
let mut absence = None;
heal_set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("reconstruction started");
let put = tokio::time::timeout(
Duration::from_secs(7),
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
)
.await;
release.notify_one();
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("heal finishes after releasing reconstruction")
.expect("heal task joins");
put.expect("foreground PUT finishes during reconstruction")
.expect("foreground PUT must not time out");
assert!(
matches!(heal_result, Err(Error::Timeout)),
"interrupted heal must be retryable: {heal_result:?}"
);
drop(_pause);
let mut absence = None;
let (_, retry_error) = set
.heal_object_with_retirement(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
.expect("a later heal must retry the current object");
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
let mut reader = set
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read foreground write after heal");
let mut actual = Vec::new();
reader.stream.read_to_end(&mut actual).await.expect("read full object");
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
}
#[tokio::test]
#[serial_test::serial]
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
use crate::object_api::WriteCompletion;
use std::time::Duration;
use tokio::io::AsyncReadExt;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "background-heal-version-delete";
let object = "object.bin";
set.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("create versioned bucket");
let original = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
&ObjectOptions {
versioned: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write original version");
let version = original.version_id.expect("version id").to_string();
let survivor = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x52; 1024]),
&ObjectOptions {
versioned: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write surviving version")
.version_id
.expect("surviving version id");
let metadata = disks[0]
.read_version("", bucket, object, &version, &ReadOptions::default())
.await
.expect("read original metadata");
let missing = dirs[0]
.path()
.join(bucket)
.join(object)
.join(metadata.data_dir.expect("external data").to_string())
.join("part.1");
tokio::fs::remove_file(missing).await.expect("remove shard");
let (_pause, arrived, release) =
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
let heal_set = set.clone();
let heal_version = version.clone();
let heal = tokio::spawn(async move {
heal_set
.heal_object_with_retirement(
bucket,
object,
&heal_version,
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut None,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
});
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
.await
.expect("reconstruction started");
let delete = tokio::time::timeout(
Duration::from_secs(7),
set.delete_object(
bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
),
)
.await;
release.notify_one();
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
.await
.expect("heal finishes")
.expect("heal task joins");
delete
.expect("foreground delete finishes")
.expect("foreground delete succeeds");
assert!(
matches!(heal_result, Err(Error::Timeout)),
"interrupted heal must be retryable: {heal_result:?}"
);
let result = set
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
..Default::default()
},
)
.await;
assert!(
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
"stale heal must not resurrect the deleted version: {result:?}"
);
let mut survivor = set
.get_object_reader(
bucket,
object,
None,
http::HeaderMap::new(),
&ObjectOptions {
versioned: true,
version_id: Some(survivor.to_string()),
..Default::default()
},
)
.await
.expect("surviving version remains readable");
let mut body = Vec::new();
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
assert_eq!(body, vec![0x52; 1024]);
for disk in &disks {
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
assert!(
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
"deleted version must remain absent on each disk: {physical:?}"
);
}
}
#[tokio::test]
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
+7 -449
View File
@@ -4184,19 +4184,7 @@ impl SetDisks {
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
.await
}
pub(crate) async fn put_object_with_prepared_reservations(
&self,
bucket: &str,
object: &str,
data: &mut PutObjReader,
opts: &ObjectOptions,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
.await
}
@@ -4215,7 +4203,7 @@ impl SetDisks {
// The owned publication capability supplies the exact target write
// domain at commit time; the generic PUT path must not reacquire it.
opts.no_lock = true;
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
.await
}
@@ -4226,9 +4214,8 @@ impl SetDisks {
data: &'a mut PutObjReader,
opts: &'a ObjectOptions,
publication_fence: Option<RemoteTuplePublicationFence>,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
}
async fn put_object_with_old_current_size_inner(
@@ -4238,7 +4225,6 @@ impl SetDisks {
data: &mut PutObjReader,
opts: &ObjectOptions,
mut publication_fence: Option<RemoteTuplePublicationFence>,
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
let protect_write = opts.shard_integrity_write_enabled();
@@ -4803,83 +4789,6 @@ impl SetDisks {
.await?;
}
// A new version has no outer bucket snapshot. Prepare its remote
// owners after staging, before taking lifecycle or object locks.
let prepare_new_version = reservations.is_none()
&& !opts.overwrites_existing_version()
&& !self.ctx.lock_manager().is_disabled()
&& crate::set_disk::put_rename_early_ack_enabled()
&& write_context.acquires_namespace()
&& write_context.allows_early_ack(opts.write_completion)
&& publication_fence.is_none()
&& opts.scanner_publication_commit_scope.is_none()
&& opts.decommission_capacity_admission.is_none()
&& opts.decommission_capacity.is_none()
&& opts.quota_admission.is_none()
&& opts.object_lock_config_snapshot.is_none()
&& opts.bucket_lifecycle_lock_fence.is_none()
&& opts.namespace_lock_fence.is_none()
&& !opts.data_movement
&& !is_meta_bucketname(bucket)
&& opts
.put_object_cancellation
.as_ref()
.is_none_or(|token| !token.is_cancelled())
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
.await
.is_some();
#[cfg(any(test, feature = "test-util"))]
if prepare_new_version {
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
}
// Destructive writes prepare before the bucket snapshot. Local-only
// fixtures without a metadata store can prepare before the object lock.
let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
&& crate::set_disk::put_rename_early_ack_enabled()
&& write_context.acquires_namespace()
&& write_context.allows_early_ack(opts.write_completion)
&& publication_fence.is_none()
&& opts.scanner_publication_commit_scope.is_none()
&& opts.decommission_capacity_admission.is_none()
&& !opts.data_movement
&& !is_meta_bucketname(bucket)
{
if let Some(prepared) = reservations {
crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
} else if prepare_new_version && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil()) {
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
&shuffle_disks,
bucket,
object,
incarnation,
&self.ctx,
true,
)
.await?
} else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
.await
.is_none()
&& let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
{
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
&shuffle_disks,
bucket,
object,
incarnation,
&self.ctx,
false,
)
.await?
} else {
None
}
} else {
None
};
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
#[cfg(test)]
{
@@ -4903,9 +4812,7 @@ impl SetDisks {
|| (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered)
{
#[cfg(any(test, feature = "test-util"))]
if !prepare_new_version {
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
}
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
if let Some(expected_incarnation_id) = source_bucket_incarnation_id
&& opts.bucket_lifecycle_lock_fence.is_none()
{
@@ -5361,11 +5268,6 @@ impl SetDisks {
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
let commit_object_lock_snapshot = opts
.object_lock_config_snapshot
.as_ref()
.filter(|snapshot| snapshot.owns_lifecycle_guard())
.cloned();
// A scanner publication scope owns the movement permit until the
// complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases
@@ -5379,26 +5281,6 @@ impl SetDisks {
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& commit_owns_namespace_guard
&& commit_scanner_publication_scope.is_none();
let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
|| commit_object_lock_snapshot
.as_ref()
.is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
let reservation_ctx = (commit_allows_early_ack
&& write_context.acquires_namespace()
&& commit_object_lock_guard.is_some()
&& commit_decommission_object_lock_guard.is_none()
&& commit_decommission_capacity_guard.is_none()
&& commit_publication_guard.is_none()
&& !quota_mutation_fence
&& !opts.data_movement
&& !is_meta_bucketname(bucket)
&& lifecycle_owned)
.then(|| Arc::clone(&self.ctx));
let prepared_metadata_reservations = if reservation_ctx.is_some() {
prepared_metadata_reservations
} else {
None
};
// Full-tail callers also transfer owned guards to the coordinator:
// cancelling their ACK waiter must not cancel an in-flight rename.
let detach_commit_owner = commit_scanner_publication_scope.is_some()
@@ -5481,7 +5363,7 @@ impl SetDisks {
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _decommission_capacity_guard
.as_ref()
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
.is_some_and(|guard| guard.is_lock_lost())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
@@ -5555,7 +5437,7 @@ impl SetDisks {
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| _decommission_capacity_guard
.as_ref()
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
.is_some_and(|guard| guard.is_lock_lost())
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
@@ -5722,7 +5604,6 @@ impl SetDisks {
)
.with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone())
.with_reservation_context(reservation_ctx, prepared_metadata_reservations)
.with_namespace_commit_guard(
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
),
@@ -5746,7 +5627,6 @@ impl SetDisks {
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
}
let mut rename_guard_release = None;
let mut committed_tail_guard = None;
let mut needs_immediate_heal = false;
let mut tail_owns_tmp_cleanup = false;
rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -5778,14 +5658,7 @@ impl SetDisks {
.or_else(|| commit_version_suspended.then(Uuid::nil))
.map(|version_id| version_id.to_string());
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
None
} else {
_object_lock_guard
.take()
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
};
committed_tail_guard = object_lock_guard.clone();
let object_lock_guard = _object_lock_guard.take();
let publication_guard = _publication_guard.take();
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
@@ -5810,7 +5683,6 @@ impl SetDisks {
write_context,
publication_guard,
bucket_lifecycle_guard,
commit_object_lock_snapshot,
decommission_object_lock_guard,
decommission_capacity_guard,
),
@@ -5832,7 +5704,6 @@ impl SetDisks {
write_context,
publication_guard,
bucket_lifecycle_guard,
object_lock_snapshot,
decommission_object_lock_guard,
decommission_capacity_guard,
),
@@ -5846,7 +5717,6 @@ impl SetDisks {
drop(write_context);
drop(publication_guard);
drop(bucket_lifecycle_guard);
drop(object_lock_snapshot);
cleanup_runtime::run(cleanup_reservation, async move {
let zero_target_cleanup = targets.is_empty();
rustfs_io_metrics::record_put_object_stage_duration(
@@ -6102,13 +5972,6 @@ impl SetDisks {
lock_held_post_rename_started,
);
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
if let Some(guard) = committed_tail_guard.take() {
// The committed tail still owns every physical rename. Convert
// only after metadata invalidation and before acknowledging PUT.
// Unsupported peers keep their exclusive leases and the tail
// retains ownership until its physical work has drained.
let _ = guard.lock().await.downgrade_committed_tail().await;
}
if let Some(release) = rename_guard_release.take() {
let _ = release.send(true);
}
@@ -22318,117 +22181,6 @@ mod put_object_tmp_cleanup_tests {
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-tail-lifecycle";
let object = "object";
for disk in &disks {
disk.make_volume(bucket).await.expect("create volume");
}
let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
let lifecycle = lock
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
.await
.expect("lifecycle read lock");
let metadata = lock
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
.await
.expect("metadata read lock");
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
Uuid::new_v4(),
bucket,
Uuid::new_v4(),
OffsetDateTime::now_utc(),
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
lifecycle,
metadata,
));
let opts = ObjectOptions {
object_lock_config_snapshot: Some(snapshot),
..Default::default()
};
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer_set = Arc::clone(&set);
let put = tokio::spawn(async move {
writer_set
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("rename tail pauses");
put.await.expect("PUT joins").expect("PUT acknowledged");
let mut bucket_delete =
Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
barrier.release();
drop(barrier);
let guard = match early {
Ok(result) => {
drop(result.expect("early bucket lock result"));
panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
}
Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
.await
.expect("lifecycle lock released after tail")
.expect("bucket delete acquires lock"),
};
drop(guard);
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
],
async {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "put-early-ack-tail-read";
let object = "early-ack-tail-read-object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let put_set = Arc::clone(&set_disks);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
put_set
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
.await
.expect("tail disk should pause during rename");
put.await
.expect("PUT task should join")
.expect("PUT should be acknowledged at quorum");
let result = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await;
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
let mut body = Vec::new();
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
.await
.expect("object body should not wait for the tail")
.expect("object body should drain");
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
rename_barrier.release();
drop(rename_barrier);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
@@ -22501,200 +22253,6 @@ mod put_object_tmp_cleanup_tests {
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_does_not_timeout_the_next_put() {
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-early-ack-next-writer";
let object = "object";
for disk in &disks {
disk.make_volume(bucket).await.expect("create volume");
}
{
let map = set.ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
let lifecycle = lifecycle_lock
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
.await
.expect("lifecycle read lock");
let metadata = lifecycle_lock
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
.await
.expect("metadata read lock");
let incarnation = Uuid::new_v4();
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
Uuid::new_v4(),
bucket,
incarnation,
OffsetDateTime::now_utc(),
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
lifecycle,
metadata,
));
let mut opts = ObjectOptions {
expected_bucket_incarnation_id: Some(incarnation),
object_lock_config_snapshot: Some(snapshot.clone()),
..Default::default()
};
snapshot.add_lock_fences(&mut opts);
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let first_set = Arc::clone(&set);
let first_opts = opts.clone();
let first = tokio::spawn(async move {
first_set
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("first rename paused");
first.await.expect("first PUT joins").expect("first PUT acknowledged");
let second = tokio::time::timeout(
Duration::from_secs(7),
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
)
.await;
barrier.release();
drop(barrier);
let second = second.expect("second PUT finishes").expect("second PUT must not time out");
tokio::time::timeout(Duration::from_secs(30), async {
while rename_tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("both rename tails must drain");
let mut reader = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("read newest object");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("read full body");
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
let expected_etag = second.etag.as_ref().expect("second PUT ETag");
for disk in &disks {
let info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("each disk must retain the latest version");
assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_cannot_restore_a_deleted_version() {
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-tail-version-delete";
let object = "object";
for disk in &disks {
disk.make_volume(bucket).await.expect("create volume");
}
{
let map = set.ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
let lifecycle = lock
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
.await
.expect("lifecycle read lock");
let metadata = lock
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
.await
.expect("metadata read lock");
let incarnation = Uuid::new_v4();
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
Uuid::new_v4(),
bucket,
incarnation,
OffsetDateTime::now_utc(),
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
lifecycle,
metadata,
));
let mut opts = ObjectOptions {
versioned: true,
expected_bucket_incarnation_id: Some(incarnation),
object_lock_config_snapshot: Some(snapshot.clone()),
..Default::default()
};
snapshot.add_lock_fences(&mut opts);
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put_opts = opts.clone();
let put = tokio::spawn(async move {
writer
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("rename pauses");
let version = put
.await
.expect("PUT joins")
.expect("PUT succeeds")
.version_id
.expect("version ID")
.to_string();
let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
let delete_set = Arc::clone(&set);
let mut delete_opts = opts;
delete_opts.version_id = Some(version.clone());
let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
delete_barrier.wait_until_paused().await;
delete_barrier.release();
tokio::time::timeout(Duration::from_secs(5), async {
while !delete_barrier.namespace_acquired() {
tokio::task::yield_now().await;
}
})
.await
.expect("version delete acquires namespace while old tail is paused");
barrier.release();
drop(barrier);
delete.await.expect("delete joins").expect("version delete succeeds");
tokio::time::timeout(Duration::from_secs(30), async {
while rename_tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("rename tails drain");
for disk in &disks {
assert!(
matches!(
disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
),
"late tail must not restore the deleted version"
);
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {
+1 -10
View File
@@ -2294,17 +2294,8 @@ mod tests {
drop(
lock.get_write_lock(Duration::from_secs(30))
.await
.expect("fixture object lock should be available"),
.expect("fixture rename tail should finish before checking its generation"),
);
// A quorum-acknowledged rename can release the object lock while its
// physical tail still holds the instance-wide namespace owner.
tokio::time::timeout(Duration::from_secs(30), async {
while ecstore.ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("fixture namespace owners should drain before checking its generation");
assert_eq!(
ecstore.scanner_namespace_mutation_generation(),
generation_before_put.saturating_add(3),
+5 -139
View File
@@ -45,8 +45,7 @@
use crate::error::{Result, StorageError};
use std::collections::HashMap;
use std::future::Future;
use std::sync::{Arc, Mutex, Weak};
use std::time::Duration;
use std::sync::{Arc, Mutex};
use rustfs_lock::NamespaceLockGuard;
use rustfs_lock::distributed_lock::LockLostSignal;
@@ -66,7 +65,7 @@ struct RegisteredGuard {
enum LockLossProbe {
Distributed(Arc<LockLostSignal>),
Local(Option<rustfs_lock::fast_lock::guard::FastLockGuardProbe>),
Local,
#[cfg(test)]
Test(Arc<std::sync::atomic::AtomicBool>),
}
@@ -75,19 +74,14 @@ impl LockLossProbe {
fn from_guard(guard: &NamespaceLockGuard) -> Self {
match guard.lock_lost_signal() {
Some(signal) => Self::Distributed(signal),
None => Self::Local(match guard {
NamespaceLockGuard::Fast(guard) => guard.release_probe(),
NamespaceLockGuard::Standard(_) => None,
}),
None => Self::Local,
}
}
fn is_lost(&self) -> bool {
// Registry callers hold entries before checking a local shard; shard
// operations never acquire the bucket fence registry.
match self {
Self::Distributed(signal) => signal.is_lost(),
Self::Local(probe) => probe.as_ref().is_some_and(|probe| probe.is_lost()),
Self::Local => false,
#[cfg(test)]
Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst),
}
@@ -104,74 +98,9 @@ pub(super) struct FenceRegistration {
#[derive(Default)]
pub(crate) struct BucketFenceRegistry {
entries: Mutex<HashMap<String, FenceEntry>>,
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
}
#[derive(Default)]
pub(super) struct ReservationFenceSlot {
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
}
/// Shared by short bursts of reservations, but never kept alive by the cache.
pub(super) struct ReservationFenceGuard {
publication: NamespaceLockGuard,
lifecycle: BucketIncarnationFenceGuard,
created_at: tokio::time::Instant,
_slot: Arc<ReservationFenceSlot>,
}
impl ReservationFenceGuard {
pub(super) fn new(
publication: NamespaceLockGuard,
lifecycle: BucketIncarnationFenceGuard,
slot: Arc<ReservationFenceSlot>,
) -> Self {
Self {
publication,
lifecycle,
created_at: tokio::time::Instant::now(),
_slot: slot,
}
}
pub(super) fn is_reusable(&self) -> bool {
// Stop admitting new reservations through this shared reader so a
// queued bucket writer can progress after the existing tails drain.
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
}
fn is_held(&self) -> bool {
!self.publication.is_lock_lost()
&& !self.publication.is_released()
&& !self.lifecycle.is_lock_lost()
&& self
.lifecycle
.namespace_lock_guard()
.is_some_and(|guard| !guard.is_released())
}
}
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
fn is_valid(&self) -> bool {
self.is_held()
}
}
impl BucketFenceRegistry {
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
let key = (bucket.to_owned(), incarnation);
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
return slot;
}
if slots.len() >= 1024 {
slots.retain(|_, slot| slot.strong_count() != 0);
}
let slot = Arc::new(ReservationFenceSlot::default());
slots.insert(key, Arc::downgrade(&slot));
slot
}
/// Register a new live guard for `bucket` and return the memoized
/// incarnation id if one is valid for the current coverage window.
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
@@ -305,9 +234,7 @@ impl BucketIncarnationFenceGuard {
}
pub fn is_lock_lost(&self) -> bool {
self.inner
.as_ref()
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
}
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
@@ -425,67 +352,6 @@ mod tests {
reg.exit("b", fourth.token, false);
}
#[tokio::test]
async fn disabled_local_lock_does_not_add_a_namespace_fence() {
let manager = Arc::new(rustfs_lock::GlobalLockManager::Disabled(
rustfs_lock::fast_lock::DisabledLockManager::new(),
));
let lock = NamespaceLock::with_local_manager("disabled-fence".into(), manager);
let guard = lock
.get_read_lock(ObjectKey::new("b", "object"), "owner", Duration::from_secs(1))
.await
.expect("disabled lock");
let mut opts = crate::object_api::ObjectOptions::default();
opts.add_namespace_lock_guard(&guard);
assert!(opts.namespace_lock_fence.is_none());
}
#[tokio::test]
async fn forced_local_fence_loss_invalidates_live_incarnation_memo() {
let manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
let global = Arc::new(rustfs_lock::GlobalLockManager::Enabled(manager.clone()));
let lock = NamespaceLock::with_local_manager("forced-fence".into(), global);
let key = ObjectKey::new("b", "lifecycle");
let registry = Arc::new(BucketFenceRegistry::default());
let inner = lock
.get_read_lock(key.clone(), "first", Duration::from_secs(1))
.await
.expect("first lock");
let mut first = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
let mut opts = crate::object_api::ObjectOptions::default();
first.attach_to_object_options(&mut opts);
opts.add_namespace_lock_guard(first.namespace_lock_guard().expect("local guard"));
assert!(
!opts
.bucket_lifecycle_lock_fence
.as_ref()
.expect("lifecycle fence")
.is_lock_lost()
);
assert!(!opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
assert_eq!(manager.force_unlock(&key), 1);
assert!(
opts.bucket_lifecycle_lock_fence
.as_ref()
.expect("attached fence")
.is_lock_lost()
);
assert!(opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
let writer = lock
.get_write_lock(key.clone(), "writer", Duration::from_secs(1))
.await
.expect("writer");
drop(writer);
let inner = lock
.get_read_lock(key, "second", Duration::from_secs(1))
.await
.expect("second lock");
let mut second = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
assert_eq!(second.validate(true, || async { Ok(uuid(8)) }).await.expect("reload"), uuid(8));
assert!(first.validate(true, || async { Ok(uuid(7)) }).await.is_err());
}
#[tokio::test]
async fn fence_guard_forwards_distributed_lock_loss_to_registry() {
let registry = Arc::new(BucketFenceRegistry::default());
-1
View File
@@ -456,7 +456,6 @@ mod rebalance;
pub(crate) use rebalance::PoolSpaceForTest;
pub(crate) mod utils;
pub(crate) use peer::admit_local_disk;
use peer::init_local_peer;
pub use peer::{
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
+4 -5
View File
@@ -19,6 +19,7 @@ use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use crate::storage_api_contracts::object::ObjectOperations as _;
use futures::{StreamExt, stream};
use rustfs_utils::path::encode_dir_object_ref;
use std::collections::HashSet;
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
@@ -1185,11 +1186,9 @@ impl ECStore {
let oi = if require_all_pools {
lookup_opts.metadata_cache_safe = false;
lookup_opts.suppress_read_repair = true;
if let Err(err) = check_object_args(bucket, object) {
return Some(err);
}
let encoded_object = encode_dir_object(object);
self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
// Only completion uses this branch, after validating bucket and object.
let encoded_object = encode_dir_object_ref(object);
self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
.await
.and_then(|(info, _)| {
lookup_opts.precondition_check(&info)?;
+2 -602
View File
@@ -4814,45 +4814,7 @@ impl ECStore {
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
let mut early_opts = opts.clone();
// Bucket publication precedes lifecycle and metadata transaction locks.
let prepared = if self.single_pool()
&& !self.ctx.lock_manager().is_disabled()
&& crate::set_disk::put_rename_early_ack_enabled()
&& early_opts.write_completion == crate::object_api::WriteCompletion::Quorum
&& early_opts.overwrites_existing_version()
&& !is_meta_bucketname(bucket)
&& !early_opts.no_lock
&& !early_opts.data_movement
&& early_opts.write_lock_context.is_none()
&& early_opts.object_lock_config_snapshot.is_none()
&& early_opts.bucket_lifecycle_lock_fence.is_none()
&& early_opts.namespace_lock_fence.is_none()
&& early_opts.scanner_publication_commit_scope.is_none()
&& early_opts.decommission_capacity_admission.is_none()
&& early_opts.decommission_capacity.is_none()
&& early_opts.quota_admission.is_none()
&& early_opts
.put_object_cancellation
.as_ref()
.is_none_or(|token| !token.is_cancelled())
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
{
check_new_object_args(bucket, object, false)?;
let incarnation = match early_opts.expected_bucket_incarnation_id {
Some(incarnation) => incarnation,
None => self.bucket_incarnation_id(bucket).await?,
};
early_opts.expected_bucket_incarnation_id = Some(incarnation);
let encoded = encode_dir_object(object);
let set = self.pools[0].get_disks_by_key(&encoded);
let disks = set.disks.read().await.clone();
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, &encoded, incarnation, &self.ctx, true).await?
} else {
None
};
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, &early_opts)).await?;
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
let idx = self
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
.await?;
@@ -4861,8 +4823,7 @@ impl ECStore {
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
}
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
let result =
Box::pin(self.pools[idx].put_object_with_prepared_reservations(bucket, object.as_str(), data, &opts, prepared)).await;
let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
match result {
Ok((object_info, old_current_size)) => {
@@ -9977,567 +9938,6 @@ mod tests {
}
}
#[tokio::test]
#[serial_test::serial]
async fn ecstore_early_ack_prepares_before_snapshot_and_orders_the_next_put() {
use crate::disk::ReadOptions;
use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
let set = store.pools[0].get_disks_by_key("object");
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
{
let map = ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "ecstore-reservation-order";
let object = "object";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let first_store = Arc::clone(&store);
let first = tokio::spawn(async move {
first_store
.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
&ObjectOptions::default(),
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("first tail pauses");
first.await.expect("first task").expect("first PUT");
let second = tokio::time::timeout(
Duration::from_secs(7),
store.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
&ObjectOptions::default(),
),
)
.await;
barrier.release();
drop(barrier);
let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
tokio::time::timeout(Duration::from_secs(30), async {
while tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("rename tails drain");
let expected_etag = second.etag.as_ref().expect("second ETag");
for disk in &disks {
let info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read latest metadata");
assert_eq!(info.metadata.get("etag"), Some(expected_etag));
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
}
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn ecstore_new_version_early_ack_orders_the_next_put() {
use crate::disk::ReadOptions;
use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
let set = store.pools[0].get_disks_by_key("object");
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
{
let map = ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "ecstore-versioned-reservation-order";
let object = "object";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let first_store = Arc::clone(&store);
let first = tokio::spawn(async move {
first_store
.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("first tail pauses");
first.await.expect("first task").expect("first PUT");
let second = tokio::time::timeout(
Duration::from_secs(7),
store.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
&ObjectOptions {
versioned: true,
..Default::default()
},
),
)
.await;
barrier.release();
drop(barrier);
let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
tokio::time::timeout(Duration::from_secs(30), async {
while tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("rename tails drain");
let expected_etag = second.etag.as_ref().expect("second ETag");
for disk in &disks {
let info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read latest metadata");
assert_eq!(info.metadata.get("etag"), Some(expected_etag));
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
}
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn local_prepared_reservations_hold_bucket_publication_fence() {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
let set = store.pools[0].get_disks_by_key("object");
let disks = set.disks.read().await.clone();
{
let map = ctx.local_disk_map();
let mut active = map.write().await;
for disk in disks.iter().flatten() {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "local-reservation-bucket-fence";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
let reservations =
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, "object", incarnation, &ctx, true)
.await
.expect("prepare reservations")
.expect("all disks prepared");
let lock = store
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.expect("publication lock");
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
assert!(tokio::time::timeout(Duration::from_millis(50), &mut writer).await.is_err());
drop(reservations);
let guard = tokio::time::timeout(Duration::from_secs(5), writer)
.await
.expect("reservations release fence")
.expect("writer lock");
drop(guard);
}
#[tokio::test]
#[serial_test::serial]
async fn long_staged_put_does_not_timeout_the_next_overwrite() {
use crate::set_disk::{
ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier,
};
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
let set = store.pools[0].get_disks_by_key("object");
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
{
let map = ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "long-staged-reservation";
let object = "object";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let staging_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
let tail_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&store);
let first = tokio::spawn(async move {
writer
.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
&ObjectOptions::default(),
)
.await
});
staging_barrier.wait_until_paused().await;
tokio::time::pause();
tokio::time::advance(Duration::from_secs(25)).await;
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(26)).await;
tokio::task::yield_now().await;
tokio::time::resume();
staging_barrier.release();
tokio::time::timeout(Duration::from_secs(30), tail_barrier.wait_until_paused())
.await
.expect("first tail pauses");
first.await.expect("first task").expect("first PUT");
drop(staging_barrier);
let second = tokio::time::timeout(
Duration::from_secs(7),
store.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
&ObjectOptions::default(),
),
)
.await;
tail_barrier.release();
drop(tail_barrier);
assert!(matches!(second, Ok(Ok(_))), "second PUT after a long staging interval failed: {second:?}");
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn bucket_reservations_do_not_exhaust_shared_lock_holders() {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
let set = store.pools[0].get_disks_by_key("object");
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
{
let map = ctx.local_disk_map();
let mut active = map.write().await;
for disk in &disks {
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
}
}
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "reservation-reader-capacity";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
let disk = disks[0].endpoint().to_string();
let mut tokens = Vec::new();
let mut failure = None;
for index in 0..256 {
match store
.reserve_local_object_metadata_at_incarnation(
&disk,
bucket,
&format!("object-{index}"),
incarnation,
Duration::from_secs(1),
)
.await
{
Ok(token) => tokens.push(token),
Err(error) => {
failure = Some((index, error));
break;
}
}
}
for token in tokens {
assert!(
store
.cancel_local_object_metadata_reservation(&disk, token)
.await
.expect("cancel token")
);
}
assert!(failure.is_none(), "bucket reservations exhausted shared lock holders: {failure:?}");
}
#[tokio::test]
#[serial_test::serial]
async fn expired_reservation_fence_does_not_bypass_a_waiting_bucket_writer() {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
let set = store.pools[0].get_disks_by_key("object");
let disk = set.disks.read().await.iter().flatten().next().expect("local disk").clone();
ctx.local_disk_map()
.write()
.await
.insert(disk.endpoint().to_string(), Some(disk.clone()));
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "reservation-writer-fairness";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
let disk_ref = disk.endpoint().to_string();
let first = store
.reserve_local_object_metadata_at_incarnation(&disk_ref, bucket, "first", incarnation, Duration::from_secs(5))
.await
.expect("first reservation");
let lock = store
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.expect("publication lock");
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
assert!(
tokio::time::timeout(Duration::from_millis(20), &mut writer).await.is_err(),
"writer waits for first token"
);
tokio::time::pause();
tokio::time::advance(Duration::from_secs(2)).await;
tokio::time::resume();
let mut second = Box::pin(store.reserve_local_object_metadata_at_incarnation(
&disk_ref,
bucket,
"second",
incarnation,
Duration::from_secs(5),
));
assert!(
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
"expired cached reader must not bypass writer"
);
assert!(
store
.cancel_local_object_metadata_reservation(&disk_ref, first)
.await
.expect("cancel first token")
);
let guard = tokio::time::timeout(Duration::from_secs(2), writer)
.await
.expect("writer proceeds")
.expect("writer lock");
assert!(
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
"new reader waits while writer holds the lock"
);
drop(guard);
let token = tokio::time::timeout(Duration::from_secs(2), second)
.await
.expect("reader proceeds")
.expect("second reservation");
assert!(
store
.cancel_local_object_metadata_reservation(&disk_ref, token)
.await
.expect("cancel second token")
);
}
#[test]
#[serial_test::serial]
fn forced_capacity_probe_cannot_publish_after_a_new_pool_writer() {
std::thread::Builder::new()
.stack_size(32 * 1024 * 1024)
.spawn(|| {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime")
.block_on(async {
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_dirs, first) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let (_other_dirs, second) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first, second], Arc::clone(&ctx)).await);
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "forced-capacity-probe";
let object = "object";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeQuotaRename);
let put_store = Arc::clone(&store);
let put = tokio::spawn(async move {
put_store
.handle_put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![1; 8193]),
&ObjectOptions::default(),
)
.await
});
barrier.wait_until_paused().await;
let manager = ctx.lock_manager().as_fast_lock_manager().expect("local lock manager");
let key = rustfs_lock::ObjectKey::new(RUSTFS_META_BUCKET, crate::core::pools::POOL_META_NAME);
assert_eq!(manager.force_unlock(&key), 1, "force release capacity probe");
let lock = store.pools[0]
.new_ns_lock(RUSTFS_META_BUCKET, crate::core::pools::POOL_META_NAME)
.await
.expect("pool lock");
let writer = lock.get_write_lock(Duration::from_secs(5)).await.expect("new pool writer");
barrier.release();
let result = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("PUT finishes")
.expect("PUT task");
assert!(
matches!(&result, Err(StorageError::NamespaceLockQuorumUnavailable { .. })),
"a PUT must reject a lost capacity probe: {result:?}"
);
drop(writer);
assert!(
matches!(
store.handle_get_object_info(bucket, object, &ObjectOptions::default()).await,
Err(StorageError::ObjectNotFound(..))
),
"the rejected PUT must not publish an object"
);
});
})
.expect("test thread")
.join()
.expect("test completes");
}
#[tokio::test]
async fn force_released_local_bucket_fence_cannot_activate_a_prepared_reservation() {
for released in ["publication", "lifecycle"] {
let fast = Arc::new(rustfs_lock::FastObjectLockManager::new());
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(fast.clone()));
let lock = rustfs_lock::NamespaceLock::with_local_manager("reservation-force".into(), manager);
let publication_key = rustfs_lock::ObjectKey::new("bucket", "publication");
let publication = lock
.get_read_lock(publication_key.clone(), "reservation", Duration::from_secs(5))
.await
.expect("publication lock");
let lifecycle = lock
.get_read_lock(rustfs_lock::ObjectKey::new("bucket", "lifecycle"), "reservation", Duration::from_secs(5))
.await
.expect("lifecycle lock");
let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default());
let lifecycle =
crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true);
let slot = bucket_registry.reservation_slot("bucket", Uuid::new_v4());
let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot));
let registry = crate::disk::os::MetadataReservationRegistry::new(1);
let path = std::path::Path::new("bucket/object");
let disk = Uuid::new_v4();
let token = registry
.prepare_with_owner(path, disk, Some(owner))
.expect("prepare reservation");
let released_key = rustfs_lock::ObjectKey::new("bucket", released);
assert_eq!(fast.force_unlock(&released_key), 1, "force release {released}");
let replacement = lock
.get_write_lock(released_key, "bucket-delete", Duration::from_secs(5))
.await
.expect("bucket writer acquires after forced release");
assert!(
registry.activate(token, path, disk).await.is_err(),
"a released fence cannot activate a write"
);
drop(replacement);
}
}
#[tokio::test]
async fn lost_bucket_fence_cannot_activate_a_prepared_reservation() {
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
rustfs_lock::FastObjectLockManager::new(),
)));
let client = Arc::new(RefreshLossLockClient::with_manager(manager));
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
"reservation-loss".to_string(),
vec![client.clone() as Arc<dyn rustfs_lock::LockClient>],
1,
);
let acquire = |object: &'static str| {
rustfs_lock::LockRequest::new(
rustfs_lock::ObjectKey::new("bucket", object),
rustfs_lock::LockType::Shared,
"reservation-loss",
)
.with_ttl(Duration::from_secs(2))
.with_refresh_interval(Duration::from_millis(20))
};
let publication = lock
.acquire_guard(&acquire("publication"))
.await
.expect("publication request")
.expect("publication quorum");
let signal = publication.lock_lost_signal().expect("distributed loss signal");
let lifecycle = lock
.acquire_guard(&acquire("lifecycle"))
.await
.expect("lifecycle request")
.expect("lifecycle quorum");
let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default());
let lifecycle =
crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true);
let incarnation = Uuid::new_v4();
let slot = bucket_registry.reservation_slot("bucket", incarnation);
let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot));
let registry = crate::disk::os::MetadataReservationRegistry::new(1);
let key = std::path::Path::new("bucket/object");
let disk = Uuid::new_v4();
let token = registry
.prepare_with_owner(key, disk, Some(owner))
.expect("prepare reservation");
client.reject_refreshes();
tokio::time::timeout(Duration::from_secs(5), signal.notified())
.await
.expect("publication lock loss");
assert!(
registry.activate(token, key, disk).await.is_err(),
"a lost bucket fence must not enter the mutation queue"
);
}
#[tokio::test]
#[serial_test::serial]
async fn borrowed_copy_tail_drained_waits_for_rename_tail() {
+1 -296
View File
@@ -85,190 +85,6 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
}
impl ECStore {
/// Prepare an inert user-object reservation on the currently admitted disk.
/// An unclaimed token expires and does not hold the object's metadata lock.
#[cfg(test)]
pub async fn reserve_local_object_metadata(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None)
.await
}
/// Prepare before the caller acquires bucket lifecycle or object locks.
/// The token holds the bucket publication fence through physical completion.
pub async fn reserve_local_object_metadata_at_incarnation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
incarnation: Uuid,
timeout: std::time::Duration,
) -> DiskResult<Uuid> {
if self.ctx.lock_manager().is_disabled()
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|| is_meta_bucketname(bucket)
|| incarnation.is_nil()
|| timeout.is_zero()
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
{
return Err(DiskError::InvalidPath);
}
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
let mut cached = slot.owner.lock().await;
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
Some(owner) => owner,
None => {
// Bucket deletion takes publication before lifecycle and metadata.
let lock = self
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.map_err(|error| DiskError::other(error.to_string()))?;
let publication = lock
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.map_err(|error| DiskError::other(error.to_string()))?;
// Older peers may acquire lifecycle without publication.
let lifecycle = self
.acquire_bucket_incarnation_fence(bucket, incarnation)
.await
.map_err(|error| DiskError::other(error.to_string()))?;
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
if !owner.is_reusable() {
return Err(DiskError::DiskOngoingReq);
}
*cached = Arc::downgrade(&owner);
owner
}
};
drop(cached);
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner))
.await
})
.await
.map_err(|_| DiskError::Timeout)?
}
async fn reserve_local_object_metadata_with_owner(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
timeout: std::time::Duration,
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
) -> DiskResult<Uuid> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
return Err(DiskError::InvalidPath);
}
if timeout.is_zero() {
return Err(DiskError::Timeout);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
tokio::time::timeout(
timeout.min(std::time::Duration::from_secs(5)),
wrapper.prepare_object_metadata_with_owner(bucket, object, owner),
)
.await
.map_err(|_| DiskError::Timeout)?
}
/// Refresh an inert token on its original disk and object.
pub async fn renew_local_object_metadata_reservation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
token: Uuid,
) -> DiskResult<()> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
return Err(DiskError::InvalidPath);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
wrapper.renew_object_metadata(bucket, object, token).await
}
/// Atomically enqueue a prepared token on its original disk and object.
pub async fn activate_local_object_metadata_reservation(
&self,
disk_ref: &str,
bucket: &str,
object: &str,
token: Uuid,
) -> DiskResult<()> {
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
return Err(DiskError::InvalidPath);
}
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
wrapper.activate_object_metadata(bucket, object, token).await
}
/// Cancel an unclaimed token on the currently admitted disk.
/// A token already claimed by a physical mutation cannot be cancelled.
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
Ok(wrapper.cancel_object_metadata_reservation(token))
}
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
/// The token is bound to the admitted disk instance and destination. A claimed
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
/// The caller must retain the bucket lifecycle fence until the physical tail
/// finishes; the reservation does not identify a bucket incarnation.
pub async fn rename_local_data_reserved(
&self,
disk_ref: &str,
source: (&str, &str),
fi: &FileInfo,
destination: (&str, &str),
token: Uuid,
) -> DiskResult<RenameDataResp> {
if token.is_nil()
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|| is_meta_bucketname(destination.0)
|| rustfs_utils::http::metadata_compat::contains_key_str(
&fi.metadata,
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
)
{
return Err(DiskError::InvalidPath);
}
rename_local_data_with_ctx(
&self.ctx,
disk_ref,
source,
fi,
destination,
RenameDataGuards {
metadata_reservation_token: Some(token),
..Default::default()
},
)
.await
}
/// Resolve a quota lease against this request's captured storage instance.
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
@@ -454,7 +270,6 @@ impl ECStore {
scanner_publication_lease_token: scanner_token,
external_guard,
namespace_owner: None,
..Default::default()
},
)
.await
@@ -536,7 +351,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
Ok(())
}
pub(crate) async fn admit_local_disk(
async fn admit_local_disk(
ctx: &Arc<InstanceContext>,
disk: &DiskStore,
disk_id: Option<Uuid>,
@@ -968,116 +783,6 @@ mod tests {
.expect("seeded metadata bytes")
}
#[tokio::test]
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
let ctx = Arc::new(InstanceContext::new());
let dir = tempfile::tempdir().expect("disk root");
let id = Uuid::new_v4();
let disk = target_disk(&ctx, dir.path(), id).await;
let disk_ref = disk.endpoint().to_string();
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let version = Uuid::new_v4();
let fi = target_file_info("object", version, b"reserved");
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
.await
.expect("user bucket");
assert!(matches!(
store
.reserve_local_object_metadata(
&disk_ref,
"target-bucket/../.rustfs.sys",
"object",
std::time::Duration::from_secs(5),
)
.await,
Err(DiskError::InvalidPath)
));
assert!(matches!(
store
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
.await,
Err(DiskError::InvalidPath)
));
let token = store
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
.await
.expect("reserve current disk");
assert!(matches!(
store
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
.await,
Err(DiskError::InvalidPath)
));
store
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
.await
.expect("activate token");
store
.rename_local_data_reserved(
&disk_ref,
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
&fi,
("target-bucket", "object"),
token,
)
.await
.expect("reserved publication");
assert!(
!store
.cancel_local_object_metadata_reservation(&disk_ref, token)
.await
.expect("consumed token")
);
let read = disk
.read_version(
"target-bucket",
"target-bucket",
"object",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read reserved publication");
assert_eq!(read.data, fi.data);
let stale = store
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
.await
.expect("reserve before disk replacement");
let replacement = target_disk(&ctx, dir.path(), id).await;
assert!(matches!(
store
.rename_local_data_reserved(
&disk_ref,
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
&fi,
("target-bucket", "object"),
stale,
)
.await,
Err(DiskError::DiskOngoingReq)
));
let after = replacement
.read_version(
"target-bucket",
"target-bucket",
"object",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("replacement retains publication");
assert_eq!(after.data, fi.data);
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
}
#[tokio::test]
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
for warm in [false, true] {
-68
View File
@@ -27,7 +27,6 @@ use futures::{StreamExt, stream::FuturesUnordered};
use metrics::{counter, gauge};
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
use rustfs_madmin::heal_commands::HealResultItem;
use std::collections::HashMap;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
@@ -1551,7 +1550,6 @@ impl ErasureSetHealer {
let checkpoint = checkpoint_manager.get_checkpoint().await;
let page_resume_index = *current_object_index;
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
let mut object_locks: HashMap<String, Arc<tokio::sync::Mutex<()>>> = HashMap::new();
let mut page_tasks = FuturesUnordered::new();
let mut completed_in_page = 0usize;
@@ -1723,7 +1721,6 @@ impl ErasureSetHealer {
let set_label = set_disk_id.to_string();
let heal_opts = self.heal_opts;
let semaphore = semaphore.clone();
let object_lock = object_locks.entry(object_name.clone()).or_default().clone();
let target_endpoints = self.target_endpoints.clone();
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
@@ -1738,13 +1735,6 @@ impl ErasureSetHealer {
.as_ref()
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
let work = async move {
// Versions share one namespace lock. Queue them before taking a
// page permit so a waiting version cannot block other objects.
let _object_guard = tokio::select! {
biased;
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
guard = object_lock.lock() => guard,
};
let mut retries = 0;
let mut admin_identity = admin_identity;
let mut admin_task_id = admin_task_id;
@@ -4057,64 +4047,6 @@ mod resume_loop_tests {
assert_eq!(unique.len(), 50, "no version healed twice");
}
#[tokio::test]
async fn page_serializes_versions_without_blocking_other_objects() {
temp_env::async_with_vars(
[
(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("true")),
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
],
async {
let mut env = make_env().await;
env.healer.source = HealRequestSource::AutoHeal;
env.healer = env
.healer
.with_replacement_targets(vec!["replacement-a".into()], Some(env.task_id.clone()));
let gate = Arc::new(tokio::sync::Semaphore::new(0));
*env.storage.heal_gate.lock().unwrap() = Some(gate.clone());
for (name, version) in [("a", "v1"), ("a", "v2"), ("b", "v1")] {
env.storage
.set_result(name, Some(version), replacement_target_ok_result("replacement-a", name));
}
env.storage.set_page(
None,
Page {
items: vec![
item("a", Some("v1"), false),
item("a", Some("v2"), false),
item("b", Some("v1"), false),
],
next: None,
truncated: false,
},
);
let observe = async {
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
let notified = env.storage.heal_started.notified();
if env.storage.calls().len() >= 2 {
break;
}
notified.await;
}
})
.await
.is_ok();
let calls = env.storage.calls();
gate.add_permits(3);
assert!(started, "different objects should run concurrently");
assert_eq!(calls.len(), 2);
assert!(calls.iter().any(|(name, _)| name == "a"));
assert!(calls.iter().any(|(name, _)| name == "b"));
};
let ((processed, successful, failed, skipped, result), ()) = tokio::join!(run(&env), observe);
result.expect("all versions heal");
assert_eq!((processed, successful, failed, skipped), (3, 3, 0, 0));
},
)
.await;
}
#[tokio::test]
async fn auto_replacement_runs_two_object_repairs_within_the_configured_limit() {
temp_env::async_with_vars(
+1 -9
View File
@@ -21,9 +21,6 @@ const MAX_DEFERRED_OBJECTS: usize = 256;
const MAX_DEFERRED_BYTES: usize = 256 * 1024;
const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
// A recovered pool can become writable just after the ordinary retry budget.
// Keep this extra attempt inside the existing deferred age and backoff caps.
const MAX_SLOWDOWN_OBJECT_RETRIES: u32 = MAX_BUCKET_OBJECT_HEAL_RETRIES + 1;
struct DeferredObject {
name: String,
@@ -915,12 +912,7 @@ impl HealTask {
);
} else if !age_exhausted
&& err.is_recoverable_object_heal()
&& retry_attempt
< if matches!(&err, Error::Storage(crate::heal::StorageError::SlowDown)) {
MAX_SLOWDOWN_OBJECT_RETRIES
} else {
MAX_BUCKET_OBJECT_HEAL_RETRIES
}
&& retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES
{
terminal_outcome = false;
debug!(
@@ -220,29 +220,6 @@ async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unproc
}
}
#[tokio::test(start_paused = true)]
async fn slowdown_can_recover_within_existing_deferred_age_budget() {
let storage = Arc::new(pages_storage(&[&["a"]]));
storage
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert("a".to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
let task = bucket_task(storage.clone());
let started = tokio::time::Instant::now();
task.execute()
.await
.expect("transient slowdown recovers within the deferred age budget");
assert!(started.elapsed() < Duration::from_secs(30));
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 5);
let outcome = task.get_outcome().await;
assert_eq!(
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(1, 0, 4)
);
assert_eq!(outcome.objects.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
@@ -251,7 +228,7 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
.heal_object_outcomes
.lock()
.expect("outcomes")
.insert(name.to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
.insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
}
let task = bucket_task(storage.clone());
let execution = task.execute();
@@ -262,9 +239,9 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
let outcome = task.get_outcome().await;
assert_eq!(
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(4, 4, 20)
(4, 4, 16)
);
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 20);
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
for name in ["a", "b", "c", "d"] {
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
}
-62
View File
@@ -382,42 +382,6 @@ impl LockClient for LocalClient {
}
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let shard = self.get_shard(lock_id);
let mut guards = shard.write().await;
let Some(entry) = guards.get_mut(lock_id) else {
return Ok(false);
};
if entry.is_expired() {
return Ok(false);
}
Ok(entry.guard.downgrade_committed_tail())
}
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let shard = self.get_shard(lock_id);
let mut guards = shard.write().await;
let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
if entry.is_expired() {
return Ok(false);
}
Ok(entry.guard.try_upgrade_committed_tail())
}
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
self.refresh(lock_id).await
}
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
let shard = self.get_shard(lock_id);
let guards = shard.read().await;
let Some(entry) = guards.get(lock_id) else { return Ok(None) };
if entry.is_expired() {
return Ok(None);
}
Ok(entry.guard.waiting_writer())
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
self.release(lock_id).await
}
@@ -583,32 +547,6 @@ mod tests {
assert!(!response.success);
}
#[tokio::test]
async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
assert!(client.acquire_lock(&first).await.expect("initial lock").success);
assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
assert!(
client
.downgrade_committed_tail(&first.lock_id)
.await
.expect("retry downgrade")
);
let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
.with_acquire_timeout(Duration::from_millis(500));
assert!(client.acquire_lock(&reader).await.expect("reader").success);
let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
assert!(client.release(&first.lock_id).await.expect("release tail"));
assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
assert!(client.release(&reader.lock_id).await.expect("release reader"));
assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
assert!(client.release(&writer.lock_id).await.expect("release writer"));
assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
}
#[tokio::test]
async fn acquire_rejects_a_lock_id_for_another_resource() {
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
-27
View File
@@ -20,12 +20,6 @@ use async_trait::async_trait;
use futures::future::join_all;
use std::sync::Arc;
/// Marker exchanged in a refresh request and response for a committed-tail conversion.
pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
/// Lock client trait
#[async_trait]
pub trait LockClient: Send + Sync + std::fmt::Debug {
@@ -54,27 +48,6 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
/// Refresh lock
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
/// Convert an exclusive lease to a reader-compatible committed-tail lease.
/// Unsupported peers retain their original lease and report `false`.
async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
Ok(false)
}
/// Try to restore exclusive ownership, retaining the writer fence on failure.
async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
Ok(false)
}
/// Check support without changing the lease mode.
async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
Ok(false)
}
/// Observe foreground writer contention, or return `None` when unsupported.
async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
Ok(None)
}
/// Force release lock
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
+1 -75
View File
@@ -262,8 +262,6 @@ pub struct DistributedLockGuard {
/// All underlying leases that should be refreshed and released with the guard.
entries: Vec<HeldLockEntry>,
lock_type: LockType,
/// Shared and exclusive quorums must intersect before a tail may convert.
committed_tail_quorums_intersect: bool,
/// If true, Drop will not try to release (used if user manually released).
disarmed: bool,
/// Background lease task for renewable leases.
@@ -279,13 +277,7 @@ impl DistributedLockGuard {
/// - `entries` is the full list of underlying leases that should be refreshed and released
/// with this guard.
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
fn new(
lock_id: LockId,
entries: Vec<HeldLockEntry>,
lock_type: LockType,
heartbeat: LockHeartbeatConfig,
committed_tail_quorums_intersect: bool,
) -> Self {
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
record_lock_held_acquire(lock_type);
let lock_lost = Arc::new(LockLostSignal::default());
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
@@ -304,7 +296,6 @@ impl DistributedLockGuard {
lock_id,
entries,
lock_type,
committed_tail_quorums_intersect,
disarmed: false,
refresh_task,
lock_lost,
@@ -421,70 +412,6 @@ impl DistributedLockGuard {
&self.lock_id
}
/// Convert retained peer leases without ever releasing writer exclusion.
/// A peer without conversion support keeps its exclusive lease. Partial
/// conversion is safe: both lease modes continue to exclude writers.
pub async fn downgrade_committed_tail(&mut self) -> bool {
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
{
return false;
}
let results = futures::future::join_all(
self.entries
.iter()
.map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
)
.await;
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
}
/// Require every retained lease to confirm exclusivity before committing.
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
return false;
}
let results = futures::future::join_all(
self.entries
.iter()
.map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
)
.await;
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
}
pub async fn supports_heal_lock_transition(&self) -> bool {
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
{
return false;
}
let results = futures::future::join_all(
self.entries
.iter()
.map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
)
.await;
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
}
/// Observe contention on every held lease. Unknown peers prevent a false idle result.
pub async fn waiting_writer(&self) -> Option<bool> {
if self.disarmed || self.is_lock_lost() {
return None;
}
let results =
futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
if self.is_lock_lost() {
return None;
}
if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
return Some(true);
}
results
.iter()
.all(|result| matches!(result, Ok(Some(false))))
.then_some(false)
}
/// Actual granted peer identities, which may differ from the logical guard ID.
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
self.entries.iter().map(|entry| &entry.lock_id)
@@ -672,7 +599,6 @@ impl DistributedLock {
individual_locks,
request.lock_type,
heartbeat,
self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
)))
} else {
// Check if it's a timeout or quorum failure
+5 -109
View File
@@ -19,8 +19,8 @@ use crate::fast_lock::{
use rustfs_io_metrics::{
record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Weak};
/// Global counter for guard IDs to prevent double-release
static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1);
@@ -39,25 +39,10 @@ pub struct FastLockGuard {
shard: Option<Arc<LockShard>>, // None when locks are disabled
released: bool,
disabled: bool, // True when locks are disabled globally
committed_tail: bool,
sealed_for_commit: bool,
/// Unique ID for this guard instance to prevent double-release
guard_id: u64,
}
/// A non-owning check of one local acquisition, including forced release.
#[derive(Clone)]
pub struct FastLockGuardProbe {
shard: Weak<LockShard>,
guard_id: u64,
}
impl FastLockGuardProbe {
pub fn is_lost(&self) -> bool {
self.shard.upgrade().is_none_or(|shard| !shard.is_guard_active(self.guard_id))
}
}
impl FastLockGuard {
pub(crate) fn new(key: ObjectKey, mode: LockMode, owner: Arc<str>, shard: Arc<LockShard>) -> Self {
let guard_id = GUARD_ID_COUNTER.fetch_add(1, Ordering::Relaxed);
@@ -69,8 +54,6 @@ impl FastLockGuard {
shard: Some(shard),
released: false,
disabled: false,
committed_tail: false,
sealed_for_commit: false,
guard_id,
}
}
@@ -85,8 +68,6 @@ impl FastLockGuard {
shard: None,
released: false,
disabled: true,
committed_tail: false,
sealed_for_commit: false,
guard_id,
}
}
@@ -106,83 +87,6 @@ impl FastLockGuard {
&self.owner
}
/// The physical acquisition was force-released before registration.
pub(crate) fn abandon_unregistered(&mut self) {
if !self.released {
self.released = true;
record_lock_held_release(self.mode);
}
}
/// Keep excluding writers while permitting reads of the committed version.
pub fn downgrade_committed_tail(&mut self) -> bool {
if self.released || self.sealed_for_commit {
return false;
}
if self.committed_tail {
return true;
}
if self.mode != LockMode::Exclusive {
return false;
}
if self.disabled {
self.mode = LockMode::Shared;
self.committed_tail = true;
return true;
}
let Some(shard) = &self.shard else { return false };
if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
return false;
}
record_lock_held_release(LockMode::Exclusive);
record_lock_held_acquire(LockMode::Shared);
self.mode = LockMode::Shared;
self.committed_tail = true;
true
}
/// Try to restore exclusive ownership without releasing the writer fence.
pub fn try_upgrade_committed_tail(&mut self) -> bool {
if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
return false;
}
if !self.disabled {
let Some(shard) = &self.shard else { return false };
if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
return false;
}
}
if self.committed_tail {
record_lock_held_release(LockMode::Shared);
record_lock_held_acquire(LockMode::Exclusive);
}
self.mode = LockMode::Exclusive;
self.committed_tail = false;
self.sealed_for_commit = true;
true
}
/// Observe writers waiting on this guard's exact object key.
pub fn waiting_writer(&self) -> Option<bool> {
if self.released {
return None;
}
if self.disabled {
return Some(false);
}
self.shard
.as_ref()?
.waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
}
fn release_physical(&self, shard: &LockShard) -> bool {
if self.committed_tail {
shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
} else {
shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
}
}
/// Manually release the lock early
///
/// Returns true if the lock was successfully released, false if it was
@@ -202,7 +106,7 @@ impl FastLockGuard {
}
if let Some(shard) = &self.shard {
let success = self.release_physical(shard);
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
if success {
self.released = true;
record_lock_held_release(self.mode);
@@ -234,17 +138,9 @@ impl FastLockGuard {
);
}
/// Check if the lock has been released, including by an administrator.
/// Check if the lock has been released
pub fn is_released(&self) -> bool {
self.released || self.shard.as_ref().is_some_and(|shard| !shard.is_guard_active(self.guard_id))
}
/// Observe this acquisition without extending the lock's lifetime.
pub fn release_probe(&self) -> Option<FastLockGuardProbe> {
self.shard.as_ref().map(|shard| FastLockGuardProbe {
shard: Arc::downgrade(shard),
guard_id: self.guard_id,
})
self.released
}
/// Check if this guard represents a disabled lock
@@ -273,7 +169,7 @@ impl Drop for FastLockGuard {
fn drop(&mut self) {
if let Some(shard) = &self.shard {
if !self.released && !self.disabled {
let success = self.release_physical(shard);
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
if !success {
// For high-concurrency scenarios, this is likely due to:
// 1. Lock was already released by another thread
+33 -16
View File
@@ -96,14 +96,11 @@ impl FastObjectLockManager {
/// Acquire object lock
pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
let shard = self.get_shard(&request.key);
match shard.acquire_guarded(&request).await {
Ok(acquired) => {
let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
match shard.acquire_lock(&request).await {
Ok(()) => {
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
// Register guard to prevent premature cleanup
if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
guard.abandon_unregistered();
return Err(LockResult::Timeout);
}
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
Ok(guard)
}
Err(err) => Err(err),
@@ -204,15 +201,31 @@ impl FastObjectLockManager {
let mut all_failed = Vec::new();
let mut guards = Vec::new();
for (_, requests) in shard_groups {
for (shard_id, requests) in shard_groups {
let shard = self.shards[*shard_id].clone();
for request in requests {
let key = request.key.clone();
match self.acquire_lock(request.clone()).await {
Ok(guard) => {
all_successful.push(key);
guards.push(guard);
let owner = request.owner.clone();
let mode = request.mode;
let acquired = if shard.try_fast_path_only(request) {
true
} else {
match shard.acquire_lock(request).await {
Ok(()) => true,
Err(err) => {
all_failed.push((key.clone(), err));
false
}
}
Err(err) => all_failed.push((key, err)),
};
if acquired {
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
all_successful.push(key);
guards.push(guard);
}
}
}
@@ -232,10 +245,14 @@ impl FastObjectLockManager {
let mut acquired_guards = Vec::new();
let mut failed_locks = Vec::new();
'outer: for (_, requests) in shard_groups {
'outer: for (shard_id, requests) in shard_groups {
let shard = self.shards[*shard_id].clone();
for request in requests {
match self.acquire_lock(request.clone()).await {
Ok(guard) => {
match shard.acquire_lock(request).await {
Ok(()) => {
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
acquired_guards.push(guard);
}
Err(err) => {
-1
View File
@@ -94,7 +94,6 @@ impl ObjectLockState {
pub fn reset_for_reuse(&mut self) {
// Reset atomic state
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
self.force_epoch.store(0, Ordering::Relaxed);
// Clear owners
*self.current_owner.write() = None;
+49 -500
View File
@@ -15,7 +15,6 @@
use parking_lot::RwLock;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::{Duration, Instant, SystemTime};
use tokio::time::timeout;
@@ -31,12 +30,6 @@ struct ActiveGuardInfo {
key: ObjectKey,
mode: LockMode,
owner: Arc<str>,
committed_tail: bool,
}
pub(crate) struct AcquiredLock {
state: Arc<ObjectLockState>,
force_epoch: u64,
}
#[derive(Debug, PartialEq, Eq, Hash)]
@@ -53,8 +46,6 @@ enum TestPausePoint {
BeforeFastWrite,
FastWriteExisting,
SlowLookup,
ForceAfterTailRelease,
ReleaseAfterUnregister,
}
#[cfg(test)]
@@ -78,7 +69,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
/// Lock shard to reduce global contention.
///
/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
/// When both are needed, lock `objects` before a state's owner metadata.
#[derive(Debug)]
pub struct LockShard {
/// Object lock states - using parking_lot for better performance
@@ -142,16 +133,12 @@ impl LockShard {
/// Acquire lock with fast path optimization
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
self.acquire_guarded(request).await.map(|_| ())
}
pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
let start_time = Instant::now();
// Try fast path first
if let Some(acquired) = self.try_fast_path(request) {
if let Some(_state) = self.try_fast_path(request) {
self.metrics.record_fast_path_success();
return Ok(acquired);
return Ok(());
}
// Slow path with waiting
@@ -170,7 +157,7 @@ impl LockShard {
}
/// Try fast path lock acquisition (lock-free when possible)
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
// First try to get existing state without write lock
{
let objects = self.objects.read();
@@ -181,16 +168,14 @@ impl LockShard {
test_pause(TestPausePoint::FastExisting, &request.key);
// Try atomic acquisition
let force_epoch = state.force_epoch.load(Ordering::Acquire);
let success = force_epoch != u64::MAX
&& match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
let success = match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
drop(objects);
if success {
return Some(AcquiredLock { state, force_epoch });
return Some(state);
}
}
}
@@ -210,11 +195,10 @@ impl LockShard {
#[cfg(test)]
test_pause(TestPausePoint::FastWriteExisting, &request.key);
let force_epoch = state.force_epoch.load(Ordering::Acquire);
let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
drop(objects);
if acquired {
return Some(AcquiredLock { state, force_epoch });
return Some(state);
}
} else {
// Create new state from pool and acquire immediately
@@ -222,7 +206,7 @@ impl LockShard {
let state = Arc::new(*state_box);
if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
objects.insert(request.key.clone(), state.clone());
return Some(AcquiredLock { state, force_epoch: 0 });
return Some(state);
}
}
}
@@ -231,7 +215,7 @@ impl LockShard {
}
/// Slow path with async waiting
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
// Use adaptive timeout based on current load and request priority
let adaptive_timeout = self.calculate_adaptive_timeout(request);
let deadline = start_time + adaptive_timeout;
@@ -250,7 +234,7 @@ impl LockShard {
loop {
// Keep the state in the map until its acquisition attempt completes.
let (state, success, force_epoch) = {
let (state, success) = {
let mut objects = self.objects.write();
let state = match objects.get(&request.key) {
Some(state) => state.clone(),
@@ -265,19 +249,17 @@ impl LockShard {
#[cfg(test)]
test_pause(TestPausePoint::SlowLookup, &request.key);
let force_epoch = state.force_epoch.load(Ordering::Acquire);
let success = force_epoch != u64::MAX
&& match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
let success = match request.mode {
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
};
drop(objects);
(state, success, force_epoch)
(state, success)
};
if success {
self.metrics.record_slow_path_success();
return Ok(AcquiredLock { state, force_epoch });
return Ok(());
}
// Check timeout
@@ -399,10 +381,14 @@ impl LockShard {
/// Release lock with guard ID tracking for double-release prevention
pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
// Keep force-unlock from releasing this holder and admitting a new
// same-owner holder between guard removal and physical release.
let mut guards = self.active_guards.lock();
if guards.remove(&guard_id).is_none() {
// First, try to remove the guard from active set
let guard_was_active = {
let mut guards = self.active_guards.lock();
guards.remove(&guard_id).is_some()
};
// If guard was not active, this is a double-release attempt
if !guard_was_active {
tracing::debug!(
"Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
key,
@@ -413,9 +399,6 @@ impl LockShard {
return false;
}
#[cfg(test)]
test_pause(TestPausePoint::ReleaseAfterUnregister, key);
// Proceed with normal release
let should_cleanup;
let result;
@@ -440,8 +423,6 @@ impl LockShard {
}
}
drop(guards);
if should_cleanup {
self.schedule_cleanup(key.clone());
}
@@ -449,110 +430,21 @@ impl LockShard {
result
}
/// Convert an active exclusive guard without opening a writer gap.
pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
let mut guards = self.active_guards.lock();
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
return false;
}
let objects = self.objects.read();
let Some(state) = objects.get(key) else { return false };
if !state.downgrade_committed_tail(owner) {
return false;
}
info.mode = LockMode::Shared;
info.committed_tail = true;
true
}
pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
let mut guards = self.active_guards.lock();
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
return false;
}
if info.mode == LockMode::Exclusive && !info.committed_tail {
return true;
}
if info.mode != LockMode::Shared || !info.committed_tail {
return false;
}
let objects = self.objects.read();
let Some(state) = objects.get(key) else { return false };
if !state.try_upgrade_committed_tail(owner) {
return false;
}
info.mode = LockMode::Exclusive;
info.committed_tail = false;
true
}
pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
let guards = self.active_guards.lock();
let info = guards.get(&guard_id)?.as_ref()?;
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
return None;
}
let objects = self.objects.read();
let state = objects.get(key)?;
Some(state.atomic_state.writer_observed_during_tail())
}
/// Release a converted guard and its special reader-admission state.
pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
let mut guards = self.active_guards.lock();
let Some(Some(info)) = guards.get(&guard_id) else { return false };
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
return false;
}
let (success, should_cleanup) = {
let objects = self.objects.read();
let Some(state) = objects.get(key) else { return false };
let success = state.release_committed_tail(owner);
(success, success && !state.is_locked() && !state.atomic_state.has_waiters())
};
if success {
guards.remove(&guard_id);
self.metrics.record_release();
}
drop(guards);
if should_cleanup {
self.schedule_cleanup(key.clone());
}
success
}
/// Register a guard to prevent premature cleanup
pub fn register_guard(&self, guard_id: u64) {
self.active_guards.lock().insert(guard_id, None);
}
pub(crate) fn register_guard_with_info(
&self,
guard_id: u64,
key: &ObjectKey,
mode: LockMode,
owner: &Arc<str>,
acquired: &AcquiredLock,
) -> bool {
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
let mut guards = self.active_guards.lock();
let objects = self.objects.read();
if !objects.get(key).is_some_and(|state| {
Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
}) {
return false;
}
guards.insert(
guard_id,
Some(ActiveGuardInfo {
key: key.clone(),
mode,
owner: owner.clone(),
committed_tail: false,
}),
);
true
}
/// Unregister a guard (called when guard is dropped)
@@ -568,8 +460,9 @@ impl LockShard {
guards.len()
}
/// Check whether this exact guard still owns its acquisition.
pub(crate) fn is_guard_active(&self, guard_id: u64) -> bool {
/// Check if a guard is active (for testing)
#[cfg(test)]
pub fn is_guard_active(&self, guard_id: u64) -> bool {
let guards = self.active_guards.lock();
guards.contains_key(&guard_id)
}
@@ -826,57 +719,27 @@ impl LockShard {
/// Returns the number of owners that were released. Used by the admin
/// force-unlock path to clear a stuck resource.
pub fn force_release_all(&self, key: &ObjectKey) -> usize {
// Serialize tail conversion and prevent new acquisitions while taking
// and releasing the owner snapshot. A new tail must never be mistaken
// for an ordinary shared owner by force unlock.
let mut guards = self.active_guards.lock();
let objects = self.objects.write();
let Some(state) = objects.get(key) else { return 0 };
// Acquisitions completed before this point but not yet registered
// must not later create guards for the released generation.
let _ = state
.force_epoch
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
let tail = guards.iter().find_map(|(&id, info)| {
info.as_ref()
.filter(|info| info.committed_tail && info.key == *key)
.map(|info| (id, info.owner.clone()))
});
let mut released = 0;
if let Some((id, owner)) = tail
&& state.release_committed_tail(&owner)
{
guards.remove(&id);
self.metrics.record_release();
released += 1;
}
#[cfg(test)]
test_pause(TestPausePoint::ForceAfterTailRelease, key);
let mut owners_modes = Vec::new();
if let Some(info) = state.current_owner.read().clone() {
owners_modes.push((info.owner, LockMode::Exclusive, 1));
}
for entry in state.shared_owners.read().iter() {
owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
}
for (owner, mode, count) in owners_modes {
let mut owner_released = false;
for _ in 0..count {
let success = match mode {
LockMode::Shared => state.release_shared(&owner),
LockMode::Exclusive => state.release_exclusive(&owner),
};
if success {
self.metrics.record_release();
owner_released = true;
}
let owners_modes: Vec<(Arc<str>, LockMode)> = {
let objects = self.objects.read();
let Some(state) = objects.get(key) else {
return 0;
};
let mut pairs = Vec::new();
if let Some(info) = state.current_owner.read().clone() {
pairs.push((info.owner, LockMode::Exclusive));
}
if owner_released {
for entry in state.shared_owners.read().iter() {
pairs.push((entry.owner.clone(), LockMode::Shared));
}
pairs
};
let mut released = 0;
for (owner, mode) in owners_modes {
if self.release_lock(key, &owner, mode) {
released += 1;
}
}
// Old guard drops must not release a later holder with the same owner.
guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
released
}
@@ -1116,320 +979,6 @@ mod tests {
)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
for reader in [false, true] {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-release-gap");
let old = if reader {
manager.acquire_read_lock(key.clone(), "same-owner").await
} else {
manager.acquire_write_lock(key.clone(), "same-owner").await
}
.expect("old holder");
let shard = manager.get_shard(&key).clone();
let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
let old_drop = tokio::task::spawn_blocking(move || drop(old));
entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
let can_force = shard.active_guards.try_lock().is_some();
let mut force = {
let manager = manager.clone();
let key = key.clone();
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
};
let mut new_guard = if can_force {
assert_eq!(force.await.expect("force join"), 1);
let mut guard = manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("new writer");
if reader {
assert!(guard.downgrade_committed_tail());
}
resume.send(()).expect("resume old release");
old_drop.await.expect("old release join");
guard
} else {
assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
resume.send(()).expect("resume old release");
old_drop.await.expect("old release join");
assert_eq!(force.await.expect("force join"), 0);
manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("new writer")
};
if reader && !can_force {
assert!(new_guard.downgrade_committed_tail());
}
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"an old release must not unlock the new writer"
);
drop(new_guard);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn force_unlock_does_not_capture_a_new_committed_tail() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-tail-race");
let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
assert!(old.downgrade_committed_tail());
let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
let force = {
let manager = manager.clone();
let key = key.clone();
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
};
entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
let mut next = {
let manager = manager.clone();
let key = key.clone();
tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
};
assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
resume.send(()).expect("resume force unlock");
assert_eq!(force.await.expect("force unlock task"), 1);
let mut new = next.await.expect("new writer task").expect("new writer");
assert!(new.downgrade_committed_tail());
drop(old);
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"the new committed tail must still exclude writers"
);
}
#[tokio::test]
async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "upgrade-tail");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
let reader = manager
.acquire_read_lock(key.clone(), "reader")
.await
.expect("reader during tail");
assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
.await
.is_err(),
"new readers must not starve a pending upgrade"
);
drop(reader);
assert!(tail.try_upgrade_committed_tail());
assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
assert!(
!tail.downgrade_committed_tail(),
"a delayed downgrade must not reopen reads after upgrade"
);
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
.await
.is_err()
);
drop(tail);
let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
}
#[tokio::test]
async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "brief-writer-observation");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
.await
.is_err()
);
assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
drop(tail);
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
}
#[tokio::test]
async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert!(tail.try_upgrade_committed_tail());
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
.await
.is_err()
);
assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
drop(tail);
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
assert_eq!(next.waiting_writer(), Some(false));
}
#[tokio::test]
async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "waiting-writer-observation");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert_eq!(tail.waiting_writer(), Some(false));
let writer = {
let manager = manager.clone();
let key = key.clone();
tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
};
tokio::time::timeout(Duration::from_secs(2), async {
while tail.waiting_writer() != Some(true) {
tokio::task::yield_now().await;
}
})
.await
.expect("writer becomes observable");
assert_eq!(manager.force_unlock(&key), 1);
assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
}
#[tokio::test]
async fn dropping_a_pending_upgrade_allows_readers_again() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "cancel-upgrade");
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
assert!(tail.downgrade_committed_tail());
let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
assert!(!tail.try_upgrade_committed_tail());
let mut waiting = {
let manager = manager.clone();
let key = key.clone();
tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
};
assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
drop(tail);
let second = tokio::time::timeout(Duration::from_secs(2), waiting)
.await
.expect("reader wakes after cancellation")
.expect("reader task joins")
.expect("reader obtains lock");
drop((reader, second));
}
#[tokio::test]
async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "upgrade-stale");
let mut tail = manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("write lock");
assert!(tail.downgrade_committed_tail());
assert_eq!(manager.force_unlock(&key), 1);
let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
assert!(!tail.try_upgrade_committed_tail());
}
#[tokio::test]
async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "late-downgrade");
let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
assert!(guard.try_upgrade_committed_tail());
assert!(!guard.downgrade_committed_tail());
assert!(guard.try_upgrade_committed_tail());
}
#[tokio::test]
async fn force_unlock_old_readers_cannot_release_a_new_tail() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-old-readers");
let old1 = manager
.acquire_read_lock(key.clone(), "same-owner")
.await
.expect("first reader");
let old2 = manager
.acquire_read_lock(key.clone(), "same-owner")
.await
.expect("second reader");
assert!(!old1.is_released());
assert!(!old2.is_released());
assert_eq!(manager.force_unlock(&key), 1);
let mut tail = manager
.acquire_write_lock(key.clone(), "same-owner")
.await
.expect("new writer");
assert!(tail.downgrade_committed_tail());
assert!(old1.is_released());
assert!(old2.is_released());
assert!(!tail.is_released());
drop((old1, old2));
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"old reader drops must not release the new tail"
);
}
#[tokio::test]
async fn forced_unregistered_reader_cannot_release_a_new_tail() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "force-unregistered-reader");
let shard = manager.get_shard(&key).clone();
let owner: Arc<str> = Arc::from("same-owner");
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
let acquired = shard
.acquire_guarded(&request)
.await
.expect("old reader acquired before registration");
assert_eq!(manager.force_unlock(&key), 1);
let mut tail = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("new writer");
assert!(tail.downgrade_committed_tail());
let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
old.abandon_unregistered();
drop(old);
assert!(
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
.await
.is_err(),
"an old reader must not release a new committed tail"
);
}
#[tokio::test]
async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
let manager = test_manager();
let key = ObjectKey::new("bucket", "unrelated-acquisition");
let shard = manager.get_shard(&key).clone();
let owner: Arc<str> = Arc::from("owner");
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
let other = (0..1000)
.map(|index| ObjectKey::new("bucket", format!("other-{index}")))
.find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
.expect("another key on the same shard");
let other_guard = manager
.acquire_write_lock(other.clone(), "other")
.await
.expect("other writer");
assert_eq!(manager.force_unlock(&other), 1);
drop(other_guard);
let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
drop(guard);
assert!(manager.acquire_write_lock(key, "next").await.is_ok());
}
fn start_acquisition(
manager: &Arc<FastObjectLockManager>,
request: ObjectLockRequest,
+8 -180
View File
@@ -23,11 +23,6 @@ use crate::fast_lock::types::{LockMode, LockPriority};
/// Optimized atomic lock state encoding in u64
/// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
const WRITER_FLAG_MASK: u64 = 0x1;
// A committed write tail is a reader-compatible owner that still excludes writers.
const COMMITTED_TAIL_FLAG: u64 = 0x2;
const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
const READERS_SHIFT: u8 = 8;
const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
const READERS_WAITING_SHIFT: u8 = 16;
@@ -36,6 +31,7 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
// Fast path check masks
const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
const COMPLETELY_UNLOCKED: u64 = 0;
/// Fast atomic lock state for single version
@@ -70,8 +66,8 @@ impl AtomicLockState {
let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
match mode {
LockMode::Shared => {
(state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
&& ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
// No writer and no waiting writers
(state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
}
LockMode::Exclusive => {
// Completely unlocked
@@ -87,11 +83,8 @@ impl AtomicLockState {
loop {
let current = self.state.load(Ordering::Acquire);
// While a committed tail owns a shared slot, let reads through
// even if a subsequent writer is waiting for that tail to drain.
if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
|| ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
{
// Fast path check - cannot acquire if there's a writer or writers waiting
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
return false;
}
@@ -144,78 +137,6 @@ impl AtomicLockState {
}
}
/// Atomically retain writer exclusion while allowing reads of a committed tail.
fn downgrade_committed_tail(&self) -> bool {
loop {
let current = self.state.load(Ordering::Acquire);
if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
return false;
}
let next = (current & !WRITER_FLAG_MASK)
| COMMITTED_TAIL_FLAG
| (1 << READERS_SHIFT)
| if current & WRITERS_WAITING_MASK != 0 {
TAIL_WRITER_OBSERVED_FLAG
} else {
0
};
if self
.state
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
self.update_access_time();
return true;
}
}
}
/// Restore exclusive ownership only after every other reader has left.
fn try_upgrade_committed_tail(&self) -> bool {
loop {
let current = self.state.load(Ordering::Acquire);
if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
return false;
}
let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
let next = if readers_drained {
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
| WRITER_FLAG_MASK
| TAIL_TRACK_WRITERS_FLAG
} else {
current | TAIL_UPGRADE_PENDING_FLAG
};
if self
.state
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
self.update_access_time();
return readers_drained;
}
}
}
/// Release the committed tail's own shared slot and restore ordinary fairness.
fn release_committed_tail(&self) -> bool {
loop {
let current = self.state.load(Ordering::Acquire);
if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
return false;
}
let next =
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
if self
.state
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
.is_ok()
{
self.update_access_time();
return true;
}
}
}
/// Release shared lock
pub fn release_shared(&self) -> bool {
loop {
@@ -248,7 +169,7 @@ impl AtomicLockState {
return false; // No exclusive lock to release
}
let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
let new_state = current & !WRITER_FLAG_MASK;
if self
.state
@@ -315,10 +236,7 @@ impl AtomicLockState {
return false; // Max waiting writers
}
let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
new_state |= TAIL_WRITER_OBSERVED_FLAG;
}
let new_state = current + (1 << WRITERS_WAITING_SHIFT);
if self
.state
@@ -403,15 +321,10 @@ impl AtomicLockState {
}
#[cfg(test)]
pub(crate) fn writers_waiting_count(&self) -> u16 {
pub fn writers_waiting_count(&self) -> u16 {
let state = self.state.load(Ordering::Acquire);
self.writers_waiting(state)
}
pub(crate) fn writer_observed_during_tail(&self) -> bool {
let state = self.state.load(Ordering::Acquire);
self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
}
}
/// Object lock state with version support - optimized memory layout
@@ -421,8 +334,6 @@ pub struct ObjectLockState {
// First cache line: Most frequently accessed data
/// Atomic state for fast operations
pub atomic_state: AtomicLockState,
/// Changes whenever an administrator force-releases this state.
pub(crate) force_epoch: AtomicU64,
// Second cache line: Notification mechanisms
/// Notification for readers (traditional)
@@ -466,7 +377,6 @@ impl ObjectLockState {
pub fn new() -> Self {
Self {
atomic_state: AtomicLockState::new(),
force_epoch: AtomicU64::new(0),
read_notify: Notify::new(),
write_notify: Notify::new(),
optimized_notify: OptimizedNotify::new(),
@@ -515,70 +425,6 @@ impl ObjectLockState {
true
}
/// Convert the current exclusive owner into the committed-tail owner.
pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
let mut current = self.current_owner.write();
let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
return false;
};
let mut shared = self.shared_owners.write();
if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
return false;
}
shared.push(SharedOwnerEntry {
owner: info.owner,
count: 1,
acquired_at: info.acquired_at,
lock_timeout: info.lock_timeout,
});
*current = None;
drop(shared);
drop(current);
self.optimized_notify.notify_readers();
true
}
pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
let mut current = self.current_owner.write();
let mut shared = self.shared_owners.write();
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
return false;
};
if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
return false;
}
let info = shared.remove(pos);
*current = Some(ExclusiveOwnerInfo {
owner: info.owner,
acquired_at: info.acquired_at,
lock_timeout: info.lock_timeout,
});
true
}
/// Release the committed-tail owner's shared slot without releasing readers.
pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
let mut shared = self.shared_owners.write();
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
return false;
};
if !self.atomic_state.release_committed_tail() {
return false;
}
if shared[pos].count > 1 {
shared[pos].count -= 1;
} else {
shared.remove(pos);
}
let no_readers = shared.is_empty();
drop(shared);
if no_readers {
self.optimized_notify.notify_writer();
}
self.optimized_notify.notify_readers();
true
}
/// Release shared lock
pub fn release_shared(&self, owner: &Arc<str>) -> bool {
let mut shared = self.shared_owners.write();
@@ -709,24 +555,6 @@ mod tests {
assert!(!state.release_exclusive());
}
#[test]
fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
let state = AtomicLockState::new();
assert!(state.try_acquire_exclusive());
assert!(state.inc_writers_waiting());
assert!(state.downgrade_committed_tail());
assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
assert!(state.release_committed_tail());
assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
assert!(state.release_shared());
assert!(state.try_acquire_exclusive());
assert!(state.release_exclusive());
state.dec_writers_waiting();
assert!(state.is_free());
}
// Regression for the waiter-preserving exclusive CAS.
//
// The acquisition CAS used to demand a fully-zero state word, which
-61
View File
@@ -111,67 +111,6 @@ mod fast_lock_tests {
assert!(guard.is_released(), "Guard should be marked as released");
}
#[tokio::test]
async fn force_unlock_old_tail_cannot_release_new_tail() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "force-tail-generation");
let owner: Arc<str> = Arc::from("same-owner");
let mut old = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("old writer");
assert!(old.downgrade_committed_tail());
assert_eq!(manager.force_unlock(&key), 1);
let mut new = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("new writer");
assert!(new.downgrade_committed_tail());
drop(old);
let attempt =
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
.await;
assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
drop(new);
}
#[tokio::test]
async fn committed_tail_allows_reader_while_writer_waits() {
let manager = Arc::new(create_test_manager());
let key = ObjectKey::new("test-bucket", "tail-handoff");
let mut tail = manager
.acquire_write_lock(key.clone(), Arc::from("first-writer"))
.await
.expect("initial writer");
let writer = tokio::spawn({
let manager = Arc::clone(&manager);
let key = key.clone();
async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
});
tokio::time::timeout(Duration::from_secs(2), async {
while manager.current_lock_counts().1 == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("second writer should be queued");
assert!(tail.downgrade_committed_tail());
let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
.await
.expect("reader should not wait behind committed tail")
.expect("reader should acquire");
assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
drop(tail);
assert!(!writer.is_finished(), "reader still excludes the next writer");
drop(reader);
let writer = tokio::time::timeout(Duration::from_secs(2), writer)
.await
.expect("writer should resume")
.expect("writer task should join")
.expect("writer should acquire");
drop(writer);
}
#[tokio::test]
async fn current_lock_counts_report_live_holders_and_waiters() {
let manager = Arc::new(create_test_manager());
-32
View File
@@ -107,38 +107,6 @@ impl NamespaceLockGuard {
}
}
/// Permit reads while continuing to exclude competing writers.
pub async fn downgrade_committed_tail(&mut self) -> bool {
match self {
Self::Standard(guard) => guard.downgrade_committed_tail().await,
Self::Fast(guard) => guard.downgrade_committed_tail(),
}
}
/// Try to restore exclusive access while retaining the writer fence.
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
match self {
Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
Self::Fast(guard) => guard.try_upgrade_committed_tail(),
}
}
/// Check whether the held leases can safely use heal lock transitions.
pub async fn supports_heal_lock_transition(&self) -> bool {
match self {
Self::Standard(guard) => guard.supports_heal_lock_transition().await,
Self::Fast(_) => true,
}
}
/// Observe whether a writer is waiting on this object's held leases.
pub async fn waiting_writer(&self) -> Option<bool> {
match self {
Self::Standard(guard) => guard.waiting_writer().await,
Self::Fast(guard) => guard.waiting_writer(),
}
}
/// Manually release the lock early
pub fn release(&mut self) -> bool {
match self {
-317
View File
@@ -154,43 +154,6 @@ impl crate::client::LockClient for DelayedClient {
}
}
#[derive(Debug)]
struct PendingTailClient(LocalClient);
#[async_trait::async_trait]
impl crate::client::LockClient for PendingTailClient {
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
self.0.acquire_lock(request).await
}
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
self.0.release(lock_id).await
}
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
self.0.refresh(lock_id).await
}
async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
std::future::pending().await
}
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
self.0.force_release(lock_id).await
}
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
self.0.check_status(lock_id).await
}
async fn get_stats(&self) -> crate::Result<LockStats> {
self.0.get_stats().await
}
async fn close(&self) -> crate::Result<()> {
self.0.close().await
}
async fn is_online(&self) -> bool {
self.0.is_online().await
}
async fn is_local(&self) -> bool {
self.0.is_local().await
}
}
#[derive(Debug)]
struct FlakyAcquireClient {
inner: LocalClient,
@@ -616,286 +579,6 @@ async fn test_namespace_lock_guard_lock_id() {
}
}
#[tokio::test]
async fn tail_conversion_rejects_non_intersecting_quorums() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
let resource = ObjectKey::new("bucket", "non-intersecting-tail");
let mut tail = lock
.get_write_lock(resource, "writer", Duration::from_secs(2))
.await
.expect("write lease");
assert!(
!tail.downgrade_committed_tail().await,
"conversion requires intersecting read and write quorums"
);
assert!(tail.release());
}
#[tokio::test]
async fn partial_tail_conversion_keeps_writers_excluded() {
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
for index in 0..4 {
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
if index < 2 {
clients.push(inner);
} else {
clients.push(Arc::new(DelayedClient {
inner,
delay: Duration::ZERO,
}));
}
}
let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
let resource = ObjectKey::new("bucket", "partial-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"mixed leases still exclude writers"
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "second", Duration::from_secs(3))
.await
.expect("writer after release");
assert!(writer.release());
}
#[tokio::test]
async fn cancelled_tail_conversion_keeps_writers_excluded() {
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
let resource = ObjectKey::new("bucket", "cancel-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(
tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
.await
.is_err()
);
let mut reader = lock
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
.await
.expect("other peers converted before cancellation");
assert!(reader.release());
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"cancelled conversion must still exclude writers"
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "second", Duration::from_secs(3))
.await
.expect("writer after release");
assert!(writer.release());
}
#[tokio::test]
async fn unsupported_peer_keeps_exclusive_tail_lease() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| {
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
Arc::new(DelayedClient {
inner,
delay: Duration::ZERO,
}) as Arc<dyn crate::client::LockClient>
})
.collect();
let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
let resource = ObjectKey::new("bucket", "old-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
assert!(
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
.await
.is_err()
);
assert!(
lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
.await
.is_err()
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "writer", Duration::from_secs(3))
.await
.expect("writer after tail");
assert!(writer.release());
}
#[tokio::test]
async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
let resource = ObjectKey::new("bucket", "committed-tail");
let mut tail = lock
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
.await
.expect("first writer");
assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
let mut reader = lock
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
.await
.expect("committed reader");
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"tail and reader must exclude writer"
);
assert!(tail.release());
assert!(
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
.await
.is_err(),
"reader still excludes writer"
);
assert!(reader.release());
let mut writer = lock
.get_write_lock(resource, "second", Duration::from_secs(3))
.await
.expect("writer after drain");
assert!(writer.release());
}
#[tokio::test]
async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
let resource = ObjectKey::new("bucket", "tail-upgrade");
let mut tail = lock
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
.await
.expect("write lock");
assert!(tail.supports_heal_lock_transition().await);
assert!(tail.downgrade_committed_tail().await);
let mut reader = lock
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
.await
.expect("reader");
assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
assert!(reader.release());
tokio::time::timeout(Duration::from_secs(2), async {
while !tail.try_upgrade_committed_tail().await {
tokio::task::yield_now().await;
}
})
.await
.expect("retry must finish partial upgrades after asynchronous reader release");
assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
assert!(
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
.await
.is_err()
);
assert!(tail.release());
let mut writer = lock
.get_write_lock(resource, "writer", Duration::from_secs(2))
.await
.expect("writer after release");
assert!(writer.release());
}
#[tokio::test]
async fn distributed_tail_observes_a_waiting_writer() {
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
.collect();
let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
let resource = ObjectKey::new("bucket", "writer-observation");
let mut tail = lock
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
.await
.expect("heal lock");
assert!(tail.downgrade_committed_tail().await);
assert_eq!(tail.waiting_writer().await, Some(false));
let writer = {
let lock = Arc::clone(&lock);
let resource = resource.clone();
tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
};
tokio::time::timeout(Duration::from_secs(2), async {
while tail.waiting_writer().await != Some(true) {
tokio::task::yield_now().await;
}
})
.await
.expect("waiting writer becomes visible on a held peer");
assert!(tail.release());
let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
assert!(guard.release());
}
#[tokio::test]
async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
use crate::client::LockClient;
let clients: Vec<Arc<LocalClient>> = (0..3)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
.collect();
let lock = NamespaceLock::with_clients(
"partial-upgrade-writer".to_owned(),
clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
);
let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
let mut tail = lock
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
.await
.expect("heal lock");
let ids = match &tail {
NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
};
let mut holders = Vec::new();
for client in &clients {
for id in &ids {
if client.check_status(id).await.expect("peer status").is_some() {
holders.push(Arc::clone(client));
break;
}
}
}
assert!(holders.len() >= 2, "write quorum must hold multiple peers");
assert!(tail.downgrade_committed_tail().await);
let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
assert_eq!(
tail.waiting_writer().await,
Some(true),
"partial upgrade must retain the earlier contention"
);
assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
assert!(tail.release());
}
#[tokio::test]
async fn test_namespace_lock_distributed_multi_node_simulation() {
// Simulate a 3-node distributed environment where each node has its own lock backend
File diff suppressed because it is too large Load Diff
-140
View File
@@ -1092,56 +1092,6 @@ impl_canonical_mutation_body!(
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
// the JSON fallback decode path, breaks the signature-bound digest.
pub fn canonical_metadata_reservation_request_body(
request: &proto_gen::node_service::MetadataReservationRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
body.push_str(&request.disk)?;
body.push_str(&request.bucket)?;
body.push_str(&request.object)?;
body.push_u64(request.timeout_ms);
body.push_bytes(&request.bucket_incarnation_id)?;
Ok(body.finish())
}
pub fn canonical_metadata_reservation_cancel_request_body(
request: &proto_gen::node_service::MetadataReservationCancelRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
body.push_str(&request.disk)?;
body.push_bytes(&request.token)?;
Ok(body.finish())
}
pub fn canonical_metadata_reservation_activate_request_body(
request: &proto_gen::node_service::MetadataReservationActivateRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
body.push_str(&request.disk)?;
body.push_str(&request.bucket)?;
body.push_str(&request.object)?;
body.push_bytes(&request.token)?;
// Keep ordinary activations byte-compatible with existing peers. Renewal
// adds a distinct signed marker so an older peer cannot treat it as activation.
if request.renew_only {
body.push_bool(true);
}
Ok(body.finish())
}
pub fn canonical_reserved_rename_data_request_body(
request: &proto_gen::node_service::ReservedRenameDataRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
if let Some(rename) = request.rename.as_ref() {
body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
} else {
body.push_bytes(&[])?;
}
body.push_bytes(&request.token)?;
Ok(body.finish())
}
pub fn canonical_rename_data_request_body(
request: &proto_gen::node_service::RenameDataRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -1772,96 +1722,6 @@ mod disk_mutation_canonical_tests {
}
}
#[test]
fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
use proto_gen::node_service::{
MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
ReservedRenameDataRequest,
};
let request = MetadataReservationRequest {
disk: "disk".into(),
bucket: "bucket".into(),
object: "object".into(),
timeout_ms: 5000,
bucket_incarnation_id: vec![1; 16].into(),
};
let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
for mutate in [
|r: &mut MetadataReservationRequest| r.disk = "other".into(),
|r: &mut MetadataReservationRequest| r.bucket = "other".into(),
|r: &mut MetadataReservationRequest| r.object = "other".into(),
|r: &mut MetadataReservationRequest| r.timeout_ms = 1,
|r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
] {
let mut changed = request.clone();
mutate(&mut changed);
bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
}
assert_all_distinct(&bodies);
let activation = MetadataReservationActivateRequest {
disk: "disk".into(),
bucket: "bucket".into(),
object: "object".into(),
token: vec![1; 16].into(),
renew_only: false,
};
let mut legacy_activation = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
legacy_activation.push_str(&activation.disk).expect("disk");
legacy_activation.push_str(&activation.bucket).expect("bucket");
legacy_activation.push_str(&activation.object).expect("object");
legacy_activation.push_bytes(&activation.token).expect("token");
assert_eq!(
canonical_metadata_reservation_activate_request_body(&activation).expect("ordinary activation"),
legacy_activation.finish(),
"ordinary activation must retain its previous body digest"
);
let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
for mutate in [
|r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
|r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
|r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
|r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
|r: &mut MetadataReservationActivateRequest| r.renew_only = true,
] {
let mut changed = activation.clone();
mutate(&mut changed);
bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
}
assert_all_distinct(&bodies);
let cancellation = MetadataReservationCancelRequest {
disk: "disk".into(),
token: vec![1; 16].into(),
};
let mut changed = cancellation.clone();
changed.token = vec![2; 16].into();
assert_ne!(
canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
);
let rename = ReservedRenameDataRequest {
rename: Some(RenameDataRequest {
dst_path: "object".into(),
..Default::default()
}),
token: vec![1; 16].into(),
};
let mut changed_token = rename.clone();
changed_token.token = vec![2; 16].into();
let mut changed_object = rename.clone();
changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
assert_all_distinct(&[
canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
rename: None,
token: rename.token,
})
.expect("missing rename"),
]);
}
#[test]
fn rename_data_canonical_body_binds_every_field() {
let baseline = RenameDataRequest {
-30
View File
@@ -339,32 +339,6 @@ message RenameDataResponse {
bytes rename_data_resp_bin = 4;
}
message MetadataReservationRequest {
string disk = 1;
string bucket = 2;
string object = 3;
uint64 timeout_ms = 4;
bytes bucket_incarnation_id = 5;
}
message MetadataReservationCancelRequest {
string disk = 1;
bytes token = 2;
}
message MetadataReservationActivateRequest {
string disk = 1;
string bucket = 2;
string object = 3;
bytes token = 4;
bool renew_only = 5;
}
message ReservedRenameDataRequest {
RenameDataRequest rename = 1;
bytes token = 2;
}
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
// accepts an arbitrary volume or object path.
message PoolMetaNativeRecoveryRequest {
@@ -1340,10 +1314,6 @@ service NodeService {
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
// The method itself requires source-preserving multipart publication on every peer.
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
+18 -479
View File
@@ -1826,34 +1826,6 @@ impl Node for NodeService {
self.handle_rename_data(request, false, true, false).await
}
async fn reserve_object_metadata(
&self,
request: Request<MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
self.handle_reserve_object_metadata(request).await
}
async fn cancel_object_metadata_reservation(
&self,
request: Request<MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
self.handle_cancel_object_metadata_reservation(request).await
}
async fn activate_object_metadata_reservation(
&self,
request: Request<MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
self.handle_activate_object_metadata_reservation(request).await
}
async fn rename_reserved_data(
&self,
request: Request<ReservedRenameDataRequest>,
) -> Result<Response<RenameDataResponse>, Status> {
self.handle_rename_reserved_data(request).await
}
async fn undo_pool_meta_write(
&self,
request: Request<DeleteVersionRequest>,
@@ -3273,17 +3245,15 @@ mod tests {
HealControlRequest, HealControlResponse, ListBucketRequest, ListDirRequest, ListVolumesRequest,
LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest,
LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest,
MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest, MetadataReservationCancelRequest,
MetadataReservationRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PoolMetaNativeRecoveryRequest, PreparePartTransactionRequest,
ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, ReservedRenameDataRequest, ScannerActivityRequest,
ScannerDirtyUsageSnapshotRequest, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest,
StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass, TierMutationPeerState,
TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, WriteRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest,
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -3301,12 +3271,8 @@ mod tests {
use tonic::{Request, Response, Status};
use uuid::Uuid;
const DISK_MUTATION_RPC_METHODS: [&str; 38] = [
const DISK_MUTATION_RPC_METHODS: [&str; 34] = [
"renamedata",
"reserveobjectmetadata",
"activateobjectmetadatareservation",
"cancelobjectmetadatareservation",
"renamereserveddata",
"renamemultipartdata",
"recovermultipartsource",
"renamedataatincarnation",
@@ -5004,44 +4970,6 @@ mod tests {
},
rustfs_protos::canonical_rename_data_request_body
);
assert_gated!(
reserve_object_metadata,
MetadataReservationRequest {
disk: disk.clone(),
bucket: "bucket".into(),
object: "object".into(),
timeout_ms: 5000,
bucket_incarnation_id: vec![1; 16].into(),
},
rustfs_protos::canonical_metadata_reservation_request_body
);
assert_gated!(
activate_object_metadata_reservation,
MetadataReservationActivateRequest {
disk: disk.clone(),
bucket: "bucket".into(),
object: "object".into(),
token: Uuid::new_v4().as_bytes().to_vec().into(),
renew_only: false,
},
rustfs_protos::canonical_metadata_reservation_activate_request_body
);
assert_gated!(
cancel_object_metadata_reservation,
MetadataReservationCancelRequest {
disk: disk.clone(),
token: Uuid::new_v4().as_bytes().to_vec().into(),
},
rustfs_protos::canonical_metadata_reservation_cancel_request_body
);
assert_gated!(
rename_reserved_data,
ReservedRenameDataRequest {
rename: Some(RenameDataRequest::default()),
token: Uuid::new_v4().as_bytes().to_vec().into(),
},
rustfs_protos::canonical_reserved_rename_data_request_body
);
assert_gated!(
recover_multipart_source,
RecoverMultipartSourceRequest::default(),
@@ -7547,15 +7475,11 @@ mod tests {
}
async fn target_rpc_fixture() -> TargetRpcFixture {
target_rpc_fixture_with_bucket_metadata(false).await
}
async fn target_rpc_fixture_with_bucket_metadata(initialize_bucket_metadata: bool) -> TargetRpcFixture {
super::timeout(Duration::from_secs(90), async {
let root = tempfile::tempdir().expect("target RPC root");
let env = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(root.path())
.init_bucket_metadata(initialize_bucket_metadata)
.init_bucket_metadata(false)
.build()
.await;
ObjectStore::new(env.ecstore.clone())
@@ -8222,13 +8146,6 @@ mod tests {
}
async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
stage_target_rpc_inner(fixture, true).await
}
async fn stage_target_rpc_inner(
fixture: &TargetRpcFixture,
create_volume: bool,
) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions};
let set = fixture
.env
@@ -8251,9 +8168,7 @@ mod tests {
}];
fi.data = Some(Bytes::from_static(b"target-rpc-inline"));
fi.set_inline_data();
if create_volume {
disk.make_volume("target-rpc").await.expect("target volume");
}
disk.make_volume("target-rpc").await.expect("target volume");
disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone())
.await
.expect("stage real inline body");
@@ -8305,363 +8220,6 @@ mod tests {
request
}
#[tokio::test]
async fn metadata_reservation_handlers_bind_and_publish_on_the_target_disk() {
use crate::storage::storage_api::ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions};
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("target-rpc", false).await;
let incarnation = fixture
.env
.ecstore
.bucket_incarnation_id("target-rpc")
.await
.expect("bucket incarnation");
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-stage", fi.clone())
.await
.expect("stage reserved body");
let service = make_server_for_context(Some(fixture.context.clone()));
let mut reserve = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let mut stale = Request::new(reserve.get_ref().clone());
stale.get_mut().bucket_incarnation_id = Uuid::new_v4().as_bytes().to_vec().into();
let stale_body = rustfs_protos::canonical_metadata_reservation_request_body(stale.get_ref()).expect("stale body");
set_tonic_canonical_body_digest(&mut stale, &stale_body).expect("stale digest");
mark_v2_authenticated(&mut stale);
assert!(
!service
.reserve_object_metadata(stale)
.await
.expect("stale handler")
.into_inner()
.success,
"a token must not be issued for a different bucket incarnation"
);
let mut self_lock = Request::new(reserve.get_ref().clone());
self_lock.get_mut().object = rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_owned();
let self_lock_body =
rustfs_protos::canonical_metadata_reservation_request_body(self_lock.get_ref()).expect("self-lock body");
set_tonic_canonical_body_digest(&mut self_lock, &self_lock_body).expect("self-lock digest");
mark_v2_authenticated(&mut self_lock);
assert!(
!service
.reserve_object_metadata(self_lock)
.await
.expect("self-lock handler")
.into_inner()
.success,
"a token cannot hold the same lock that its object commit must acquire"
);
let body = rustfs_protos::canonical_metadata_reservation_request_body(reserve.get_ref()).expect("reserve body");
set_tonic_canonical_body_digest(&mut reserve, &body).expect("reserve digest");
mark_v2_authenticated(&mut reserve);
let response = service
.reserve_object_metadata(reserve)
.await
.expect("reserve RPC")
.into_inner();
assert!(response.success, "reservation must succeed: {:?}", response.error);
assert_eq!(response.protocol_version, 2);
let token = response.token;
let mut tampered_renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: token.clone(),
renew_only: false,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(tampered_renew.get_ref())
.expect("ordinary activation body");
set_tonic_canonical_body_digest(&mut tampered_renew, &body).expect("ordinary activation digest");
mark_v2_authenticated(&mut tampered_renew);
tampered_renew.get_mut().renew_only = true;
assert_eq!(
service
.activate_object_metadata_reservation(tampered_renew)
.await
.expect_err("changing activation into renewal must invalidate the body digest")
.code(),
tonic::Code::PermissionDenied
);
let mut renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: token.clone(),
renew_only: true,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renew body");
set_tonic_canonical_body_digest(&mut renew, &body).expect("renew digest");
mark_v2_authenticated(&mut renew);
assert!(
service
.activate_object_metadata_reservation(renew)
.await
.expect("renew RPC")
.into_inner()
.success,
"prepared reservation should renew without activation"
);
let mut activate = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: token.clone(),
renew_only: false,
});
let body =
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
mark_v2_authenticated(&mut activate);
let activated = service
.activate_object_metadata_reservation(activate)
.await
.expect("activate RPC")
.into_inner();
assert!(activated.success, "activation must succeed: {:?}", activated.error);
let message = ReservedRenameDataRequest {
rename: Some(RenameDataRequest {
disk: disk.endpoint().to_string(),
src_volume: RUSTFS_META_TMP_BUCKET.into(),
src_path: "reserved-stage".into(),
file_info: serde_json::to_string(&fi).expect("encode metadata"),
dst_volume: "target-rpc".into(),
dst_path: "destination".into(),
..Default::default()
}),
token,
};
let body = rustfs_protos::canonical_reserved_rename_data_request_body(&message).expect("rename body");
let mut tampered = Request::new(message.clone());
set_tonic_canonical_body_digest(&mut tampered, &body).expect("rename digest");
mark_v2_authenticated(&mut tampered);
tampered.get_mut().rename.as_mut().expect("rename").dst_path = "other".into();
assert_eq!(
service
.rename_reserved_data(tampered)
.await
.expect_err("tampered object")
.code(),
tonic::Code::PermissionDenied
);
let mut rename = Request::new(message);
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
mark_v2_authenticated(&mut rename);
let response = service.rename_reserved_data(rename).await.expect("rename RPC").into_inner();
assert!(response.success, "reserved rename must succeed: {:?}", response.error);
let read = disk
.read_version(
"target-rpc",
"target-rpc",
"destination",
&fi.version_id.expect("version").to_string(),
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read committed object");
assert_eq!(read.data, fi.data);
}
#[cfg(not(windows))]
#[tokio::test]
async fn reserved_rename_rpc_cancellation_does_not_stop_physical_publication() {
use crate::storage::storage_api::{
LocalPublicationPause, LocalPublicationStage,
contract::namespace::NamespaceLocking as _,
ecstore_disk::{DiskAPI, RUSTFS_META_TMP_BUCKET, ReadOptions},
};
let fixture = target_rpc_fixture_with_bucket_metadata(true).await;
fixture.env.make_bucket("target-rpc", false).await;
let incarnation = fixture
.env
.ecstore
.bucket_incarnation_id("target-rpc")
.await
.expect("bucket incarnation");
let (disk, fi, _) = stage_target_rpc_inner(&fixture, false).await;
disk.write_metadata(RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_BUCKET, "reserved-cancel", fi.clone())
.await
.expect("stage reserved body");
let service = make_server_for_context(Some(fixture.context.clone()));
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind RPC listener");
let addr = listener.local_addr().expect("RPC listener address");
let (shutdown, stopped) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
tonic::transport::Server::builder()
.add_service(NodeServiceServer::new(service))
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
let _ = stopped.await;
})
.await
.expect("serve RPC target");
});
let mut client = NodeServiceClient::connect(format!("http://{addr}"))
.await
.expect("connect RPC client");
let mut prepare = Request::new(MetadataReservationRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
timeout_ms: 5000,
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
});
let body = rustfs_protos::canonical_metadata_reservation_request_body(prepare.get_ref()).expect("prepare body");
set_tonic_canonical_body_digest(&mut prepare, &body).expect("prepare digest");
mark_v2_authenticated(&mut prepare);
let prepared = client
.reserve_object_metadata(prepare)
.await
.expect("prepare handler")
.into_inner();
assert!(prepared.success, "prepare: {:?}", prepared.error);
let mut renew = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: prepared.token.clone(),
renew_only: true,
});
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(renew.get_ref()).expect("renew body");
set_tonic_canonical_body_digest(&mut renew, &body).expect("renew digest");
mark_v2_authenticated(&mut renew);
assert!(
client
.activate_object_metadata_reservation(renew)
.await
.expect("renew handler")
.into_inner()
.success,
"renewal over the real transport must retain the prepared token"
);
let mut activate = Request::new(MetadataReservationActivateRequest {
disk: disk.endpoint().to_string(),
bucket: "target-rpc".into(),
object: "destination".into(),
token: prepared.token.clone(),
renew_only: false,
});
let body =
rustfs_protos::canonical_metadata_reservation_activate_request_body(activate.get_ref()).expect("activate body");
set_tonic_canonical_body_digest(&mut activate, &body).expect("activate digest");
mark_v2_authenticated(&mut activate);
let activated = client
.activate_object_metadata_reservation(activate)
.await
.expect("activate handler")
.into_inner();
assert!(activated.success, "activate: {:?}", activated.error);
let mut rename = Request::new(ReservedRenameDataRequest {
rename: Some(RenameDataRequest {
disk: disk.endpoint().to_string(),
src_volume: RUSTFS_META_TMP_BUCKET.into(),
src_path: "reserved-cancel".into(),
dst_volume: "target-rpc".into(),
dst_path: "destination".into(),
file_info: serde_json::to_string(&fi).expect("encode metadata"),
..Default::default()
}),
token: prepared.token,
});
let body = rustfs_protos::canonical_reserved_rename_data_request_body(rename.get_ref()).expect("rename body");
set_tonic_canonical_body_digest(&mut rename, &body).expect("rename digest");
mark_v2_authenticated(&mut rename);
let mut pause =
LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename)
.expect("install physical pause");
let mut handler = Box::pin(client.rename_reserved_data(rename));
super::timeout(Duration::from_secs(10), async {
tokio::select! {
result = &mut handler => panic!("handler completed before physical entry: {result:?}"),
entered = pause.entered() => entered.expect("physical executor entered"),
}
})
.await
.expect("bounded physical entry");
drop(handler);
assert!(
fixture.env.ecstore.namespace_commits_pending_for_test(),
"physical publication remains active"
);
let lifecycle = fixture
.env
.ecstore
.new_ns_lock("target-rpc", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
.await
.expect("bucket publication lock");
let mut bucket_writer = Box::pin(lifecycle.get_write_lock(Duration::from_secs(10)));
let early = super::timeout(Duration::from_millis(500), &mut bucket_writer).await;
let old_lifecycle = fixture
.env
.ecstore
.new_ns_lock("target-rpc", "")
.await
.expect("bucket lifecycle lock");
let mut old_writer = Box::pin(old_lifecycle.get_write_lock(Duration::from_secs(10)));
let old_early = super::timeout(Duration::from_millis(500), &mut old_writer).await;
drop(pause);
match early {
Ok(result) => {
drop(result.expect("early lifecycle result"));
panic!("bucket publication writer acquired while the remote physical write was still paused");
}
Err(_) => drop(
super::timeout(Duration::from_secs(10), bucket_writer)
.await
.expect("lifecycle owner drains")
.expect("bucket writer acquires"),
),
}
match old_early {
Ok(result) => {
drop(result.expect("early lifecycle result"));
panic!("bucket lifecycle writer acquired while the remote physical write was still paused");
}
Err(_) => drop(
super::timeout(Duration::from_secs(10), old_writer)
.await
.expect("lifecycle owner drains")
.expect("bucket lifecycle writer acquires"),
),
}
super::timeout(Duration::from_secs(10), async {
while fixture.env.ecstore.namespace_commits_pending_for_test() {
tokio::task::yield_now().await;
}
})
.await
.expect("physical publication drains");
let read = disk
.read_version(
"target-rpc",
"target-rpc",
"destination",
&fi.version_id.expect("version").to_string(),
&ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read published object");
assert_eq!(read.data, fi.data);
drop(client);
let _ = shutdown.send(());
super::timeout(Duration::from_secs(10), server)
.await
.expect("RPC server shuts down")
.expect("RPC server task");
}
#[cfg(not(windows))]
#[tokio::test]
async fn quota_fence_handlers_commit_real_data_and_drain_before_revocation() {
@@ -9416,47 +8974,28 @@ mod tests {
))
}
async fn internal_snapshot_once(
root: &std::path::Path,
) -> std::io::Result<std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>>> {
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
let mut snapshot = std::collections::BTreeMap::new();
let mut directories = (0..4)
.map(|index| std::path::PathBuf::from(format!("disk{index}/{RUSTFS_META_BUCKET}")))
.collect::<Vec<_>>();
while let Some(relative) = directories.pop() {
let mut entries = tokio::fs::read_dir(root.join(&relative)).await?;
let mut entries = tokio::fs::read_dir(root.join(&relative))
.await
.expect("read internal snapshot directory");
snapshot.insert(relative.clone(), None);
while let Some(entry) = entries.next_entry().await? {
while let Some(entry) = entries.next_entry().await.expect("read internal snapshot entry") {
let path = relative.join(entry.file_name());
let file_type = entry.file_type().await?;
let file_type = entry.file_type().await.expect("read internal snapshot entry type");
if file_type.is_dir() {
directories.push(path);
} else {
assert!(file_type.is_file(), "fixture snapshot must contain only directories and regular files");
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await?));
snapshot.insert(path, Some(tokio::fs::read(entry.path()).await.expect("read snapshot file bytes")));
}
}
}
Ok(snapshot)
}
async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap<std::path::PathBuf, Option<Vec<u8>>> {
// Startup cleanup can remove temporary paths after the namespace
// commit drains. Capture the complete tree only once it is stable.
super::timeout(Duration::from_secs(10), async {
let mut previous = None;
loop {
match internal_snapshot_once(root).await {
Ok(snapshot) if previous.as_ref() == Some(&snapshot) => return snapshot,
Ok(snapshot) => previous = Some(snapshot),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => previous = None,
Err(error) => panic!("read internal snapshot: {error}"),
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("internal snapshot must stabilize within ten seconds")
snapshot
}
fn file_info(object: &str, version: Uuid, body: Bytes) -> FileInfo {
-159
View File
@@ -507,165 +507,6 @@ impl NodeService {
Ok(store)
}
pub(super) async fn handle_reserve_object_metadata(
&self,
request: Request<MetadataReservationRequest>,
) -> Result<Response<SnapshotLeaseResponse>, Status> {
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reservation body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("metadata reservation requires body-bound authentication"))?;
let request = request.into_inner();
if request.timeout_ms == 0 {
return Err(Status::invalid_argument("reservation timeout must be positive"));
}
let incarnation = Uuid::from_slice(&request.bucket_incarnation_id)
.ok()
.filter(|id| !id.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid bucket incarnation"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
let result = store
.reserve_local_object_metadata_at_incarnation(
&request.disk,
&request.bucket,
&request.object,
incarnation,
std::time::Duration::from_millis(request.timeout_ms.min(5000)),
)
.await;
Ok(Response::new(match result {
Ok(token) => SnapshotLeaseResponse {
success: true,
token: token.as_bytes().to_vec().into(),
protocol_version: 2,
error: None,
},
Err(error) => SnapshotLeaseResponse {
success: false,
token: Bytes::new(),
protocol_version: 2,
error: Some(error.into()),
},
}))
}
pub(super) async fn handle_activate_object_metadata_reservation(
&self,
request: Request<MetadataReservationActivateRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reservation activation body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("reservation activation requires body-bound authentication"))?;
let request = request.into_inner();
let token = Uuid::from_slice(&request.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
let result = if request.renew_only {
store
.renew_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token)
.await
} else {
store
.activate_local_object_metadata_reservation(&request.disk, &request.bucket, &request.object, token)
.await
};
Ok(Response::new(match result {
Ok(()) => SnapshotLeaseMutationResponse {
success: true,
error: None,
},
Err(error) => SnapshotLeaseMutationResponse {
success: false,
error: Some(error.into()),
},
}))
}
pub(super) async fn handle_cancel_object_metadata_reservation(
&self,
request: Request<MetadataReservationCancelRequest>,
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reservation cancellation body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("reservation cancellation requires body-bound authentication"))?;
let request = request.into_inner();
let token = Uuid::from_slice(&request.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
Ok(Response::new(
match store.cancel_local_object_metadata_reservation(&request.disk, token).await {
Ok(cancelled) => SnapshotLeaseMutationResponse {
success: cancelled,
error: None,
},
Err(error) => SnapshotLeaseMutationResponse {
success: false,
error: Some(error.into()),
},
},
))
}
pub(super) async fn handle_rename_reserved_data(
&self,
request: Request<ReservedRenameDataRequest>,
) -> Result<Response<RenameDataResponse>, Status> {
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("invalid reserved rename body"))?;
verify_tonic_canonical_body_digest(&request, &body)
.map_err(|_| Status::permission_denied("reserved rename requires body-bound authentication"))?;
let request = request.into_inner();
let token = Uuid::from_slice(&request.token)
.ok()
.filter(|token| !token.is_nil())
.ok_or_else(|| Status::invalid_argument("invalid reservation token"))?;
let rename = request.rename.ok_or_else(|| Status::invalid_argument("missing rename"))?;
if !rename.scanner_publication_lease_token.is_empty() || !rename.bucket_incarnation_id.is_empty() {
return Err(Status::invalid_argument("reserved rename does not support scanner or heal scope"));
}
let decoded = decode_rename_data_request_file_info(&rename.file_info_bin, &rename.file_info)
.map_err(|_| Status::invalid_argument("invalid reserved rename FileInfo"))?;
let LocalMutationTarget::Ready(store) = self.local_mutation_target() else {
return Err(Status::failed_precondition("storage instance is unavailable"));
};
let result = store
.rename_local_data_reserved(
&rename.disk,
(&rename.src_volume, &rename.src_path),
&decoded.value,
(&rename.dst_volume, &rename.dst_path),
token,
)
.await;
let encoded = result.and_then(|value| encode_rename_data_response_payloads(&value, decoded.from_msgpack));
Ok(Response::new(match encoded {
Ok((rename_data_resp, rename_data_resp_bin)) => RenameDataResponse {
success: true,
rename_data_resp,
rename_data_resp_bin: rename_data_resp_bin.into(),
error: None,
},
Err(error) => RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Bytes::new(),
error: Some(error.into()),
},
}))
}
pub(super) async fn handle_acquire_snapshot_lease(
&self,
request: Request<SnapshotLeaseRequest>,
-194
View File
@@ -49,91 +49,6 @@ async fn refresh_lock(
lock_client: &std::sync::Arc<dyn rustfs_lock::client::LockClient>,
args: &LockRequest,
) -> GenerallyLockResponse {
let writer_marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
if args
.metadata
.tags
.get(writer_marker)
.is_some_and(|value| value == writer_marker)
{
return match lock_client.waiting_writer(&args.lock_id).await {
Ok(Some(waiting)) => GenerallyLockResponse {
success: true,
error_info: None,
lock_info: Some(format!("{writer_marker}:{}", u8::from(waiting))),
},
Ok(None) => GenerallyLockResponse {
success: false,
error_info: None,
lock_info: None,
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
let probe_marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
if args
.metadata
.tags
.get(probe_marker)
.is_some_and(|value| value == probe_marker)
{
return match lock_client.supports_heal_lock_transition(&args.lock_id).await {
Ok(success) => GenerallyLockResponse {
success,
error_info: None,
lock_info: success.then(|| probe_marker.to_owned()),
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
let upgrade_marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
if args
.metadata
.tags
.get(upgrade_marker)
.is_some_and(|value| value == upgrade_marker)
{
return match lock_client.try_upgrade_committed_tail(&args.lock_id).await {
Ok(success) => GenerallyLockResponse {
success,
error_info: None,
lock_info: success.then(|| upgrade_marker.to_owned()),
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
if args.metadata.tags.get(marker).is_some_and(|value| value == marker) {
return match lock_client.downgrade_committed_tail(&args.lock_id).await {
Ok(true) => GenerallyLockResponse {
success: true,
error_info: None,
lock_info: Some(marker.to_owned()),
},
Ok(false) => GenerallyLockResponse {
success: false,
error_info: None,
lock_info: None,
},
Err(error) => GenerallyLockResponse {
success: false,
error_info: Some(error.to_string()),
lock_info: None,
},
};
}
match lock_client.refresh(&args.lock_id).await {
Ok(true) => GenerallyLockResponse {
success: true,
@@ -432,115 +347,6 @@ mod tests {
assert!(result.lock_info.is_none());
}
#[tokio::test]
async fn refresh_reports_waiting_writer_for_held_lease() {
use super::refresh_lock;
use rustfs_lock::client::{HEAL_WAITING_WRITER_MARKER, LockClient};
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
use std::{sync::Arc, time::Duration};
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let key = ObjectKey::new("bucket", "writer-observation-rpc");
let mut held = LockRequest::new(key.clone(), LockType::Exclusive, "heal");
assert!(client.acquire_lock(&held).await.expect("acquire heal lease").success);
held.metadata
.tags
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
let writer = {
let client = Arc::clone(&client);
tokio::spawn(async move {
let request = LockRequest::new(key, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_secs(3));
let response = client.acquire_lock(&request).await.expect("writer request");
(request, response)
})
};
let expected = format!("{HEAL_WAITING_WRITER_MARKER}:1");
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let observed = refresh_lock(&client, &held).await;
if observed.lock_info.as_deref() == Some(expected.as_str()) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("waiting writer reported");
assert!(client.release(&held.lock_id).await.expect("release heal lease"));
let (request, response) = writer.await.expect("writer task");
assert!(response.success);
assert!(client.release(&request.lock_id).await.expect("release writer"));
}
#[tokio::test]
async fn refresh_upgrade_seals_lease_against_late_conversion() {
use super::refresh_lock;
use rustfs_lock::client::{
COMMITTED_TAIL_CONVERSION_MARKER, COMMITTED_TAIL_UPGRADE_MARKER, HEAL_LOCK_TRANSITION_PROBE_MARKER,
HEAL_WAITING_WRITER_MARKER, LockClient,
};
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
use std::sync::Arc;
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let mut request = LockRequest::new(ObjectKey::new("bucket", "upgrade-rpc"), LockType::Exclusive, "owner");
assert!(client.acquire_lock(&request).await.expect("acquire lease").success);
request
.metadata
.tags
.insert(HEAL_LOCK_TRANSITION_PROBE_MARKER.into(), HEAL_LOCK_TRANSITION_PROBE_MARKER.into());
let probe = refresh_lock(&client, &request).await;
assert!(probe.success);
assert_eq!(probe.lock_info.as_deref(), Some(HEAL_LOCK_TRANSITION_PROBE_MARKER));
request.metadata.tags.clear();
request
.metadata
.tags
.insert(HEAL_WAITING_WRITER_MARKER.into(), HEAL_WAITING_WRITER_MARKER.into());
let observation = refresh_lock(&client, &request).await;
assert!(observation.success);
assert_eq!(observation.lock_info.as_deref(), Some(format!("{HEAL_WAITING_WRITER_MARKER}:0").as_str()));
request.metadata.tags.clear();
request
.metadata
.tags
.insert(COMMITTED_TAIL_UPGRADE_MARKER.into(), COMMITTED_TAIL_UPGRADE_MARKER.into());
let response = refresh_lock(&client, &request).await;
assert!(response.success);
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_UPGRADE_MARKER));
request.metadata.tags.clear();
request
.metadata
.tags
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
assert!(!refresh_lock(&client, &request).await.success, "late conversion must be rejected");
assert!(client.release(&request.lock_id).await.expect("release lease"));
}
#[tokio::test]
async fn refresh_can_convert_committed_tail_and_acknowledges_protocol_version() {
use super::refresh_lock;
use rustfs_lock::client::{COMMITTED_TAIL_CONVERSION_MARKER, LockClient};
use rustfs_lock::{LocalClient, LockRequest, LockType, ObjectKey};
use std::{sync::Arc, time::Duration};
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let resource = ObjectKey::new("bucket", "tail-conversion-rpc");
let mut request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner");
assert!(client.acquire_lock(&request).await.expect("acquire write lease").success);
request
.metadata
.tags
.insert(COMMITTED_TAIL_CONVERSION_MARKER.into(), COMMITTED_TAIL_CONVERSION_MARKER.into());
let response = refresh_lock(&client, &request).await;
assert!(response.success);
assert_eq!(response.lock_info.as_deref(), Some(COMMITTED_TAIL_CONVERSION_MARKER));
let read = LockRequest::new(resource, LockType::Shared, "reader").with_acquire_timeout(Duration::from_secs(1));
assert!(client.acquire_lock(&read).await.expect("read committed tail").success);
assert!(client.release(&read.lock_id).await.expect("release reader"));
assert!(client.release(&request.lock_id).await.expect("release tail"));
}
// B1 -- refresh_lock delegates to LocalClient::refresh for a held lock, returns
// success=true, and actually extends expires_at (inverts the "parse then drop" stub).
#[tokio::test]
+1 -1
View File
@@ -22,7 +22,7 @@ class RegistryTests(unittest.TestCase):
def test_all_workflow_responsibilities_and_suite_exits_are_wired(self):
self.assertTrue(registry.check_wiring(self.registry))
self.assertEqual(len(self.registry["suites"]), 12)
self.assertEqual(sum(item["origin"] == "baseline" for item in self.registry["workflows"]), 60)
self.assertEqual(sum(item["origin"] == "baseline" for item in self.registry["workflows"]), 61)
self.assertEqual(len(self.registry["workflows"]), len(list((registry.ROOT / ".github/workflows").glob("*.yml"))))
def test_manual_and_periodic_plan_choose_identical_responsibilities(self):