mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 16:50:45 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Hauser
|
2716250d8c |
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
|
||||
@@ -296,32 +296,6 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
|
||||
@@ -1,421 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = candidate["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -1636,53 +1636,20 @@ async fn hot_window_with_old_followers(
|
||||
let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
|
||||
let scheduler_config = run.assert_default_cadence().await?;
|
||||
let baseline = run.quiet_baseline().await?;
|
||||
// A leader-lock refresh gap during old-binary replacement can make
|
||||
// `leader()` briefly observe a different node. Poll until the coordinator
|
||||
// is stable before comparing against the measured fleet. A persistent
|
||||
// change to an old-binary node is a genuine election outcome and fails.
|
||||
let (leader, leader_pid) = if mixed {
|
||||
let expected_leader = usize::try_from(
|
||||
mixed_fleet["leader"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader")?,
|
||||
)?;
|
||||
let expected_pid = mixed_fleet["leader_pid"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader pid")?;
|
||||
let mut resolved = None;
|
||||
for _ in 0..20 {
|
||||
let candidate = run.leader().await?;
|
||||
let candidate_pid = u64::from(run.pid(candidate)?);
|
||||
if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
|
||||
resolved = Some((candidate, run.pid(candidate)?));
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
match resolved {
|
||||
Some(pair) => pair,
|
||||
None => {
|
||||
let current = match run.leader().await {
|
||||
Ok(node) => Some((node, run.pid(node).ok())),
|
||||
Err(_) => None,
|
||||
};
|
||||
return Err(format!(
|
||||
"the hot window did not retain its measured new coordinator after stabilization \
|
||||
(expected node {expected_leader} pid {expected_pid}, observed {current:?})"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let leader = run.leader().await?;
|
||||
(leader, run.pid(leader)?)
|
||||
};
|
||||
let leader = run.leader().await?;
|
||||
let leader_pid = run.pid(leader)?;
|
||||
let producer = (leader + 1) % 4;
|
||||
let producers = if fault.is_some() {
|
||||
vec![producer, (leader + 2) % 4]
|
||||
} else {
|
||||
vec![producer]
|
||||
};
|
||||
if mixed
|
||||
&& (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
|
||||
|| mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
|
||||
{
|
||||
return Err("the hot window did not retain its measured new coordinator".into());
|
||||
}
|
||||
let producer_pid = run.pid(producer)?;
|
||||
let producer_address = run.peer_address(producer);
|
||||
let [cold_a, cold_b, hot] = run.buckets.clone();
|
||||
|
||||
@@ -683,7 +683,6 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -691,7 +690,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -5210,61 +5209,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
|
||||
@@ -747,16 +747,6 @@ async fn list_path_raw_inner(
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
if !revjob_rx.is_cancelled() {
|
||||
for (i, reader) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_none()
|
||||
&& pending_entries[i].is_none()
|
||||
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
|
||||
{
|
||||
pending_entries[i] = Some(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
@@ -888,7 +878,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
if entry.name == current.name {
|
||||
// If exact match, we agree.
|
||||
if current.matches_strict_borrowed(&entry) {
|
||||
if current.matches_strict(&entry) {
|
||||
top_entries[i] = Some(entry);
|
||||
agree += 1;
|
||||
} else {
|
||||
@@ -996,9 +986,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
if !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
@@ -1021,7 +1009,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if top_entries[i].is_some() && !r.skip_current() {
|
||||
if top_entries[i].is_some() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3506,7 +3506,6 @@ mod tests {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3720,45 +3720,33 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
}
|
||||
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3768,8 +3756,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5834,13 +5822,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -28365,7 +28353,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28399,88 +28387,6 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28523,7 +28429,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28932,17 +28838,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28967,43 +28873,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
|
||||
@@ -605,7 +605,6 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -627,7 +626,6 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -636,21 +634,7 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -2398,10 +2398,6 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+188
-699
File diff suppressed because it is too large
Load Diff
@@ -175,30 +175,17 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
old_metadata: &[u8],
|
||||
sync: bool,
|
||||
) -> std::io::Result<PathBuf> {
|
||||
let Some(staging_parent) = staging_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
|
||||
};
|
||||
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
|
||||
&& let Err(err) = std::fs::write(&backup_path, old_metadata)
|
||||
{
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -369,12 +356,9 @@ fn prepare_rename_commit_and_destination_metadata_std(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(Box<RenameDataResp>),
|
||||
Committed(RenameDataResp),
|
||||
CommitErr(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
#[cfg(test)]
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
@@ -556,9 +540,7 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
|
||||
Some(Arc::new(
|
||||
pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
|
||||
))
|
||||
Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -759,6 +741,11 @@ impl LocalDisk {
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -798,11 +785,10 @@ impl LocalDisk {
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
)?;
|
||||
// An existing xl.meta that replaces an existing version slot
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
// An existing xl.meta can share versions or require rollback
|
||||
// restoration when a post-rename directory sync fails.
|
||||
if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -825,19 +811,7 @@ impl LocalDisk {
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
if no_inline
|
||||
&& sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
@@ -856,31 +830,13 @@ impl LocalDisk {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& (sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
@@ -890,157 +846,49 @@ impl LocalDisk {
|
||||
if let Err(err) = sync_result
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
|
||||
}
|
||||
|
||||
let guard = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
guard,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
std::fs::remove_file(&src_file_path_owned)?;
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some()
|
||||
&& let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}))
|
||||
{
|
||||
return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}
|
||||
}
|
||||
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
destination_was_absent,
|
||||
staged_rollback_path.as_ref(),
|
||||
&guard,
|
||||
)
|
||||
&& let Err(err) =
|
||||
fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
|
||||
{
|
||||
if no_inline {
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path_owned.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
@@ -1049,7 +897,7 @@ impl LocalDisk {
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})))
|
||||
}))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
@@ -1069,23 +917,7 @@ impl LocalDisk {
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
return Ok(resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -1093,269 +925,11 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
|
||||
std::panic::resume_unwind(panic_payload);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = *restored_fi;
|
||||
fi = restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else if let Some(publication) = multipart_publication.as_ref()
|
||||
&& no_inline
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
AbortBeforeDataRename(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
RestoreTargetAfterError(DiskError, bool),
|
||||
}
|
||||
let publication_owned = Arc::clone(publication);
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let outcome = os::run_blocking_namespace_operation_with_sync(
|
||||
mutation_lease.clone(),
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let skip_parent_owned = if has_dst_buf_owned.is_some() {
|
||||
dst_parent_owned.clone()
|
||||
} else {
|
||||
dst_volume_dir_owned.clone()
|
||||
};
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf_owned.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
if let Err(err) = xlmeta.add_version(fi_moved) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
|
||||
}
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf_owned = match xlmeta.marshal_msg() {
|
||||
Ok(buf) => buf,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
|
||||
};
|
||||
|
||||
let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
|
||||
&new_dst_buf_owned,
|
||||
has_dst_buf_owned.as_deref(),
|
||||
&dst_file_path_owned,
|
||||
&directory_guard_owned,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
// When presynced, shard payloads were synced during UploadPart and
|
||||
// prepare_and_install_head_on_thread just fsynced src_data_path after
|
||||
// writing RETIREMENT_ANCHOR.
|
||||
if sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
|
||||
&& let Err(err) = os::sync_dir_files_std(src_data_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
let directory_guard_owned = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
directory_guard_owned,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let Some(src_parent) = src_data_path.parent() else {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
|
||||
};
|
||||
let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
|
||||
directory_guard_owned,
|
||||
DiskError::Unexpected,
|
||||
));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent_owned,
|
||||
&publication_root_owned,
|
||||
&directory_guard_owned,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
|
||||
}
|
||||
if has_dst_buf_owned.is_none() {
|
||||
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir_owned) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir_limited_std(dir) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
|
||||
to_file_error(err).into(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if dir == dst_volume_dir_owned.as_path() {
|
||||
break;
|
||||
}
|
||||
ancestor = dir.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))),
|
||||
Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
match outcome {
|
||||
MultipartFusedCommitOutcome::Committed(resp) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
|
||||
MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
|
||||
if !invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
|
||||
if invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
@@ -1479,81 +1053,71 @@ impl LocalDisk {
|
||||
// after the commit rename, and a crash before the rename means the
|
||||
// PUT was never acknowledged. A metadata commit: relaxed tiers
|
||||
// leave it to the page cache.
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(all(not(test), target_os = "linux"))]
|
||||
let probe_active = false;
|
||||
if let Some(publication) = multipart_publication.as_ref() {
|
||||
pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
|
||||
}
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
// The tmp xl.meta write and the shard-file fdatasync are independent
|
||||
// (disjoint paths) and both only need to be durable before the commit
|
||||
// renames below, so run them concurrently to drop a blocking
|
||||
// round-trip from the PUT commit critical path (rustfs/backlog#922
|
||||
// step 2). The "contents durable -> rename -> dst dir fsync" ordering
|
||||
// is unchanged — both futures complete before any rename — which the
|
||||
// rename_data crash-consistency harness (backlog#935) exercises.
|
||||
//
|
||||
// Shard durability: once rename_data succeeds the write is
|
||||
// acknowledged, so data must not live only in the page cache.
|
||||
// Multipart parts were already synced during rename_part, so their
|
||||
// fdatasync here is a cheap no-op. A missing source dir is left for the
|
||||
// rename below to report through the existing rollback path. Payload
|
||||
// durability is kept by both strict and relaxed.
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = multipart_publication
|
||||
.as_ref()
|
||||
.map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
let shard_sync = async {
|
||||
if durability.syncs_data_shards()
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
#[cfg(target_os = "linux")]
|
||||
if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
if multipart_publication.is_none()
|
||||
&& let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
|
||||
let dst_file_path_ref = &dst_file_path;
|
||||
let dir_guard_ref = &commit.directory_guard;
|
||||
let lease_clone = commit.mutation_lease.clone();
|
||||
let prepare_fut = pin_multipart_future(|| {
|
||||
publication.prepare_and_install_head(
|
||||
&new_dst_buf,
|
||||
has_dst_buf.as_deref(),
|
||||
dst_file_path_ref,
|
||||
dir_guard_ref,
|
||||
lease_clone,
|
||||
)
|
||||
});
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
} else {
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = commit.source_metadata.to_path_buf();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
// Surface a tmp-meta failure first (its prior serial position), then a
|
||||
// shard-sync failure; either aborts before any rename, exactly as the
|
||||
// sequential version did.
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
let prepared_metadata_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
|
||||
None => prepared_metadata_source,
|
||||
};
|
||||
commit.directory_guard = remove_dst_base_before_commit(
|
||||
dst_path,
|
||||
@@ -1893,22 +1457,13 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
Ok(RenameDataResp {
|
||||
@@ -2108,8 +1663,13 @@ impl LocalDisk {
|
||||
#[cfg(not(windows))]
|
||||
let backup_path = {
|
||||
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata)?
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
if sync {
|
||||
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
|
||||
@@ -1000,25 +1000,6 @@ pub(super) fn inspection_with_markers(
|
||||
Ok((inspection, has_physical_data_dirs_or_markers))
|
||||
}
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Native MPU source closure has its own conditional protocol. Generic
|
||||
// object recovery must never restore its head or acquire SourceRW under
|
||||
@@ -1542,9 +1523,6 @@ impl LocalDisk {
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
}
|
||||
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
.delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
|
||||
@@ -1933,14 +1911,8 @@ impl LocalDisk {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let mut file_leases = Vec::with_capacity(5);
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
let mut file_leases = Vec::with_capacity(3);
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
|
||||
file_leases.push((path, lease));
|
||||
@@ -2160,151 +2132,119 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
let (old_dir, journal_raw) = match (|| -> Result<Option<(Option<Uuid>, Vec<u8>)>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let deleted_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| v.header.version_id.unwrap_or_default())
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
if !meta.versions.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut predecessors = BTreeSet::new();
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let Some(id) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| Uuid::parse_str(name).ok())
|
||||
.filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let prepared = regular_bytes(&entry.path().join(PREPARED))?;
|
||||
let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
}
|
||||
let surviving: HashSet<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
deleted_headers.retain(|header| !surviving.contains(header));
|
||||
let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let file_name = entry.file_name();
|
||||
let Some(name_str) = file_name.to_str() else {
|
||||
sole_transaction = false;
|
||||
continue;
|
||||
};
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let dir_path = entry.path();
|
||||
let prepared = regular_bytes(&dir_path.join(PREPARED))?;
|
||||
let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
if sole_transaction && name_str == id.hyphenated().to_string() && old_dir != Some(id) {
|
||||
let is_surviving_data_dir = original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& prepared.is_none()
|
||||
&& receipt.is_none()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
}
|
||||
}
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
sole_transaction &= predecessors.is_empty();
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: target.as_deref().map(digest),
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
let raw = journal_bytes(&journal)?;
|
||||
Ok(Some((old_dir, target, raw, journal, sole_transaction)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
}
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: None,
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
Ok(Some((old_dir, journal_bytes(&journal)?)))
|
||||
})() {
|
||||
Ok(Some(pair)) => pair,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
|
||||
let staged_dir = staged.path().to_path_buf();
|
||||
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, &buf)?;
|
||||
if sync && !buf.is_empty() {
|
||||
let staged = tempfile::Builder::new()
|
||||
.prefix("delete-preparation-")
|
||||
.tempdir_in(&staging_root)?;
|
||||
for (name, bytes) in [
|
||||
(STORAGE_FORMAT_FILE_BACKUP, buf.as_slice()),
|
||||
(PREPARED, journal_raw.as_slice()),
|
||||
] {
|
||||
let mut file = std::fs::File::create(staged.path().join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
}
|
||||
let mut written_files = Vec::with_capacity(2);
|
||||
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
|
||||
let Some(bytes) = bytes else {
|
||||
continue;
|
||||
};
|
||||
let mut file = std::fs::File::create(staged_dir.join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
written_files.push(file);
|
||||
}
|
||||
}
|
||||
for file in written_files {
|
||||
file.sync_data()?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
os::fsync_dir_std(staged.path())?;
|
||||
}
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
@@ -2317,62 +2257,27 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let staged_dir = staged.keep();
|
||||
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
|
||||
let _ = std::fs::remove_dir_all(&staged_dir);
|
||||
return Err(err);
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
std::fs::rename(staged.path(), &destination)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&file_path)?;
|
||||
}
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(old_path) = old_data_path {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let reserve_res = (|| -> std::io::Result<()> {
|
||||
if let Some(uuid) = old_dir {
|
||||
let old_path = file_path.join(uuid.to_string());
|
||||
if old_path.is_dir() {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = reserve_res {
|
||||
let _ = std::fs::remove_file(&reserved_marker);
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
if let Some(ref target_bytes) = target {
|
||||
let target_in_rollback = destination.join(TARGET);
|
||||
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
|
||||
let rename_res = std::fs::rename(&linked_xl, &xl_path);
|
||||
if rename_res.is_err() {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
}
|
||||
rename_res
|
||||
} else {
|
||||
(|| -> std::io::Result<()> {
|
||||
let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
|
||||
if sync && !target_bytes.is_empty() {
|
||||
staged_xl.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_xl.path(), &xl_path)
|
||||
})()
|
||||
};
|
||||
if let Err(err) = write_res {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
} else if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
@@ -2393,7 +2298,6 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
put_prepared_delete_receipt(destination, (journal, sole_transaction));
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
@@ -2439,26 +2343,6 @@ impl LocalDisk {
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
|
||||
&& rollback.join(PREPARED).is_file()
|
||||
{
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: false,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged.path(), &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
return Ok(Ok(Some((journal, sole_transaction, true))));
|
||||
}
|
||||
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
|
||||
Ok(Some(bytes)) => bytes,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
@@ -2476,18 +2360,12 @@ impl LocalDisk {
|
||||
{
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let target_satisfied = match journal.target_digest {
|
||||
None => match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => !exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
},
|
||||
Some(expected_target) => match regular_bytes(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(Some(bytes)) => digest(&bytes) == expected_target,
|
||||
Ok(None) => false,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
},
|
||||
let current_exists = match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
};
|
||||
if !target_satisfied {
|
||||
let unversioned_target_absent = journal.target_digest.is_none() && !current_exists;
|
||||
if !unversioned_target_absent {
|
||||
return Ok(Ok(None));
|
||||
}
|
||||
let mut sole_transaction = journal.predecessors.is_empty();
|
||||
@@ -2504,15 +2382,8 @@ impl LocalDisk {
|
||||
&& id != transaction
|
||||
&& !journal.data_dirs.contains(&id)
|
||||
{
|
||||
let dir_path = entry.path();
|
||||
let is_surviving_data_dir = journal.original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& !dir_path.join(PREPARED).exists()
|
||||
&& !dir_path.join(DELETE_QUORUM_COMMITTED_FILE).exists()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
break;
|
||||
}
|
||||
sole_transaction = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2535,7 +2406,7 @@ impl LocalDisk {
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
|
||||
Ok(Ok(Some((journal, sole_transaction, unversioned_target_absent))))
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2587,15 +2458,12 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let has_target = journal.target_digest.is_some();
|
||||
let journal_for_receipt = journal.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
@@ -2606,13 +2474,11 @@ impl LocalDisk {
|
||||
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let mut had_physical_data_dirs = false;
|
||||
for data_dir in &data_dirs {
|
||||
for base in [&root, &rollback] {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in entries {
|
||||
@@ -2653,45 +2519,23 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
if !data_dirs.is_empty() && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
}
|
||||
if has_target || had_physical_data_dirs {
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal_for_receipt,
|
||||
finished: true,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged_receipt.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_receipt.path(), &receipt_path)?;
|
||||
let _ = std::fs::File::create(rollback.join(FINISHED))?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
}
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
|
||||
ignore_not_found(std::fs::remove_file(&receipt_path))?;
|
||||
for dir_path in &prune_paths {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
|
||||
if let Err(err) = std::fs::remove_dir(dir_path) {
|
||||
if is_dir_not_empty_error(&err) {
|
||||
break;
|
||||
@@ -6438,214 +6282,6 @@ mod tests {
|
||||
b"outside bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_marker_and_publish_fast_path() {
|
||||
let (_temp, disk, info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object root");
|
||||
let transaction = Uuid::new_v4();
|
||||
let mut marker = FileInfo {
|
||||
name: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: info.mod_time.map(|t| t + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &marker, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused delete marker preparation must succeed");
|
||||
assert_eq!(fused, Some(()), "versioned delete marker must use fused path");
|
||||
assert!(root.join(transaction.to_string()).join(TARGET).is_file());
|
||||
assert!(root.join(STORAGE_FORMAT_FILE).is_file());
|
||||
|
||||
let published = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("versioned target must satisfy local publish");
|
||||
assert!(published.1, "sole transaction must be true");
|
||||
assert!(published.2, "target_satisfied must be true");
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &published.0, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
let final_meta = FileMeta::load(&fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("read final xl.meta"))
|
||||
.expect("decode final xl.meta");
|
||||
assert_eq!(final_meta.versions.len(), 2);
|
||||
assert_eq!(final_meta.versions[0].header.version_id, marker.version_id);
|
||||
marker.deleted = false;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
|
||||
let (_temp, disk, mut v1, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
// Configure v1 and v2 with distinct physical data directories so deleting v2
|
||||
// exercises both `old_dir` reservation and `is_surviving_data_dir`.
|
||||
v1.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut v1.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let v1_dir = Uuid::new_v4();
|
||||
v1.data_dir = Some(v1_dir);
|
||||
fs::create_dir_all(root.join(v1_dir.to_string()))
|
||||
.await
|
||||
.expect("create v1 dir");
|
||||
fs::write(root.join(v1_dir.to_string()).join("part.1"), b"v1-data")
|
||||
.await
|
||||
.expect("write v1 part");
|
||||
|
||||
let mut v2 = v1.clone();
|
||||
v2.version_id = Some(Uuid::new_v4());
|
||||
let v2_dir = Uuid::new_v4();
|
||||
v2.data_dir = Some(v2_dir);
|
||||
fs::create_dir_all(root.join(v2_dir.to_string()))
|
||||
.await
|
||||
.expect("create v2 dir");
|
||||
fs::write(root.join(v2_dir.to_string()).join("part.1"), b"v2-data")
|
||||
.await
|
||||
.expect("write v2 part");
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(v1.clone()).expect("add v1");
|
||||
meta.add_version(v2.clone()).expect("add v2");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused versioned delete preparation must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
assert!(
|
||||
root.join(v2_dir.to_string())
|
||||
.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"))
|
||||
.is_file(),
|
||||
"deleted version data_dir must have recovery marker"
|
||||
);
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("versioned target must satisfy local publish");
|
||||
assert!(sole_transaction, "surviving v1 data_dir must not clear sole_transaction");
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_some());
|
||||
|
||||
// Simulate interruption during finish-before-finalize right after TARGET is removed
|
||||
// (after finished receipt + FINISHED marker were persisted, before receipt_path is removed).
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: true,
|
||||
};
|
||||
fs::write(
|
||||
rollback.join(DELETE_QUORUM_COMMITTED_FILE),
|
||||
serde_json::to_vec(&finished_receipt).expect("serialize finished receipt"),
|
||||
)
|
||||
.await
|
||||
.expect("write finished receipt");
|
||||
fs::write(rollback.join(FINISHED), []).await.expect("write FINISHED");
|
||||
fs::remove_file(rollback.join(STORAGE_FORMAT_FILE_BACKUP))
|
||||
.await
|
||||
.expect("remove backup");
|
||||
fs::remove_file(rollback.join(TARGET)).await.expect("remove TARGET");
|
||||
|
||||
let inspected = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("interrupted post-TARGET state must remain inspectable without FileCorrupt");
|
||||
assert_eq!(inspected.state, DeleteRecoveryState::CommitFinished);
|
||||
|
||||
// Complete cleanup via try_finish_and_finalize_prepared_delete_local.
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed after interruption");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(v2_dir.to_string()).exists(), "deleted v2 data_dir must be removed");
|
||||
assert!(
|
||||
root.join(v1_dir.to_string()).join("part.1").is_file(),
|
||||
"surviving v1 data_dir must remain intact"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
|
||||
let (_temp, disk, mut info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
info.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut info.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let data_dir = Uuid::new_v4();
|
||||
info.data_dir = Some(data_dir);
|
||||
fs::create_dir_all(root.join(data_dir.to_string()))
|
||||
.await
|
||||
.expect("create data_dir");
|
||||
fs::write(root.join(data_dir.to_string()).join("part.1"), b"physical-payload")
|
||||
.await
|
||||
.expect("write part.1");
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(info.clone()).expect("add version");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &info, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused unversioned physical delete must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("absent target must satisfy local publish");
|
||||
assert!(sole_transaction);
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_none());
|
||||
|
||||
let rollback = root.join(transaction.to_string());
|
||||
assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
|
||||
assert!(!root.exists(), "empty object directory must be pruned");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -421,7 +421,6 @@ impl LocalDisk {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
|
||||
super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
|
||||
let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
let source = store.source.clone();
|
||||
let base = store.base.clone();
|
||||
@@ -619,9 +618,6 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
root: os::PublicationRoot,
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
|
||||
@@ -238,192 +238,9 @@ pub(super) fn read_optional_in_pinned(
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct FileStamp {
|
||||
len: u64,
|
||||
modified: Option<std::time::SystemTime>,
|
||||
#[cfg(unix)]
|
||||
ino: u64,
|
||||
}
|
||||
|
||||
impl FileStamp {
|
||||
fn from_path(path: &Path) -> Option<Self> {
|
||||
let meta = std::fs::symlink_metadata(path).ok()?;
|
||||
Some(Self {
|
||||
len: meta.len(),
|
||||
modified: meta.modified().ok(),
|
||||
#[cfg(unix)]
|
||||
ino: {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
meta.ino()
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveMultipartSourceEntry {
|
||||
raw: Bytes,
|
||||
fi: FileInfo,
|
||||
head_stamp: FileStamp,
|
||||
unsettled_parts: std::collections::HashSet<usize>,
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
fi,
|
||||
head_stamp,
|
||||
unsettled_parts: std::collections::HashSet::new(),
|
||||
parts: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
source: &Path,
|
||||
part_number: Option<usize>,
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
map.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
|
||||
return None;
|
||||
}
|
||||
Some(part)
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
if !initial && has_active_multipart_source_head(source) {
|
||||
return Ok(());
|
||||
}
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
@@ -623,7 +440,6 @@ impl SourceStore {
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
@@ -654,7 +470,7 @@ impl SourceStore {
|
||||
let terminal = terminal_record_path(&source, state);
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
|
||||
Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
@@ -723,7 +539,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -735,7 +551,6 @@ impl SourceStore {
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
@@ -767,10 +582,9 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
@@ -1283,82 +1097,6 @@ pub(super) fn load_source(
|
||||
})
|
||||
}
|
||||
|
||||
fn load_uninitialized_source(
|
||||
raw: Bytes,
|
||||
path: &str,
|
||||
disk_id: Uuid,
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let sha = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let current = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
native_source_protocol(¤t)?;
|
||||
if source_status(¤t)?.is_some()
|
||||
|| publication_identity(¤t)?.is_some()
|
||||
|| SourceConditionV1::take(&mut current.clone())?.is_some()
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-bucket")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let object = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-object")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let condition = capture_source_condition(bucket, object, path, ¤t, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
|
||||
if condition.generation.epoch != 0 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let slot = condition
|
||||
.lineage
|
||||
.geometry
|
||||
.distribution
|
||||
.iter()
|
||||
.position(|index| *index == current.erasure.index)
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
|
||||
let snapshot = Some(MultipartSourceSnapshot {
|
||||
disk_id,
|
||||
physical_slot: slot,
|
||||
lineage: condition.lineage,
|
||||
generation: condition.generation,
|
||||
revision: 0,
|
||||
phase: MultipartSourcePhase::Active,
|
||||
state_sha256: [0; 32],
|
||||
head_sha256: Some(sha),
|
||||
inventory_sha256: inventory,
|
||||
capability_sha256: None,
|
||||
publication: None,
|
||||
});
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state: None,
|
||||
head: Some(SourceHead {
|
||||
sha256: sha,
|
||||
length,
|
||||
raw: Some(raw.clone()),
|
||||
}),
|
||||
original: Some(raw),
|
||||
info: Some(current),
|
||||
snapshot,
|
||||
head_matches_state: true,
|
||||
head_matches_activation: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -1585,6 +1323,41 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
/// Only native source validation may create provenance. Ordinary TMP/heal
|
||||
/// publication can retain a valid old local anchor, never issue/rebind one.
|
||||
pub async fn write_retirement_anchor(
|
||||
&self,
|
||||
state: &SourceState,
|
||||
publication: &PublicationIdentity,
|
||||
directory: &Path,
|
||||
lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
if state.lineage != publication.lineage
|
||||
|| state.generation != publication.source_generation
|
||||
|| state.disk_id != self.disk_id
|
||||
|| state.physical_slot != self.physical_slot
|
||||
{
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: self.disk_id,
|
||||
physical_slot: self.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let bytes = encode(&record)?;
|
||||
let directory = directory.to_path_buf();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
/// This caller already owns SourceRW and destination metadata ownership.
|
||||
/// An Aborting survivor can confirm a historical target only with its
|
||||
/// independent, same-disk publication anchor. Closure never creates one.
|
||||
@@ -1683,8 +1456,6 @@ struct ValidatedPreparedPublication {
|
||||
length: u64,
|
||||
}
|
||||
|
||||
type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
|
||||
|
||||
pub(super) struct NativeMultipartPublication {
|
||||
pub store: Arc<SourceStore>,
|
||||
pub publication: PublicationIdentity,
|
||||
@@ -1692,49 +1463,22 @@ pub(super) struct NativeMultipartPublication {
|
||||
state: parking_lot::Mutex<SourceState>,
|
||||
capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
|
||||
validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
|
||||
deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
pub(super) async fn prepare_native_multipart_publication(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_native_multipart_publication_deferred(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_native_multipart_publication_inner(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
defer_persist: bool,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
// Real Complete is parts-backed, including a zero-byte last part.
|
||||
if incoming.data.is_some() || incoming.inline_data() {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
|
||||
let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
|
||||
load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
|
||||
} else {
|
||||
store.load().await?
|
||||
};
|
||||
let loaded = store.load().await?;
|
||||
let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
let original = loaded
|
||||
.head
|
||||
@@ -1825,44 +1569,7 @@ impl LocalDisk {
|
||||
};
|
||||
// Copy the small source FI to an independent immutable inode. Its
|
||||
// canonical inode is never truncated to build the destination proxy.
|
||||
let directory = staging.private_directory(publication.final_data_dir);
|
||||
let anchor_record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: store.disk_id,
|
||||
physical_slot: store.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let anchor_bytes = encode(&anchor_record)?;
|
||||
state.validate(store.disk_id, &store.path)?;
|
||||
let original_bytes = original.to_vec();
|
||||
let deferred_initial = if defer_persist {
|
||||
Some((original_bytes, directory, anchor_bytes))
|
||||
} else {
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source_dir)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
None
|
||||
};
|
||||
store.write_owned_backup_and_persist(owner, false, original, &state).await?;
|
||||
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
|
||||
let context = NativeMultipartPublication {
|
||||
store,
|
||||
@@ -1871,9 +1578,14 @@ impl LocalDisk {
|
||||
state: parking_lot::Mutex::new(state),
|
||||
capability: parking_lot::Mutex::new(Some(capability)),
|
||||
validated_prepared: parking_lot::Mutex::new(None),
|
||||
deferred_initial: parking_lot::Mutex::new(deferred_initial),
|
||||
legacy,
|
||||
};
|
||||
let directory = staging.private_directory(context.publication.final_data_dir);
|
||||
let reserved = context.state.lock().clone();
|
||||
context
|
||||
.store
|
||||
.write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
|
||||
.await?;
|
||||
Ok(context)
|
||||
}
|
||||
}
|
||||
@@ -1882,15 +1594,18 @@ impl NativeMultipartPublication {
|
||||
pub fn canonical_head(&self) -> PathBuf {
|
||||
self.store.source.join(STORAGE_FORMAT_FILE)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if encode(current_state)? != encode(&state)? || !current.head_matches_state {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
// Validate the actual persisted representation once, after FileMeta's
|
||||
// canonicalization. No unvalidated input can install a source proxy.
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
@@ -1903,297 +1618,17 @@ impl NativeMultipartPublication {
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
self.store
|
||||
.write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
|
||||
.await?;
|
||||
} else {
|
||||
self.store.persist(&state).await?;
|
||||
}
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
let prepared_path = &self.prepared_path;
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
|
||||
if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = setup_res {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub async fn prepare_and_install_head(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
let prepared_path = self.prepared_path.clone();
|
||||
let canonical = self.canonical_head();
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(&prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
|
||||
prepared_metadata_source.write_all(&new_bytes, true)?;
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub async fn prepare_and_install_head(
|
||||
self: &Arc<Self>,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let this = Arc::clone(self);
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
|
||||
{
|
||||
let guard = self.validated_prepared.lock();
|
||||
@@ -2264,13 +1699,13 @@ impl NativeMultipartPublication {
|
||||
// return an error after consuming the last claimable source heads.
|
||||
self.store.restore_exact_target_with_lease(&state, lease).await
|
||||
}
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if current.head.is_some() || encode(current_state)? != encode(&state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let (expected_sha256, expected_length) = {
|
||||
let guard = self.validated_prepared.lock();
|
||||
let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
@@ -2282,88 +1717,16 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
(validated.sha256, validated.length)
|
||||
};
|
||||
let target_dir = self.store.target_path(&self.publication)?;
|
||||
self.store
|
||||
.read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleaned.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
let snapshot = self.store.persist_with_source_sync(&state, true).await?;
|
||||
*self.state.lock() = state;
|
||||
let proof = self
|
||||
.store
|
||||
@@ -2375,21 +1738,6 @@ impl NativeMultipartPublication {
|
||||
undo_capability: capability,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finish(
|
||||
self: &Arc<Self>,
|
||||
target_lease: Arc<os::NamespaceMutationLease>,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
|
||||
@@ -207,12 +207,6 @@ pub(super) struct MultipartSourceAdmission {
|
||||
initial: bool,
|
||||
}
|
||||
impl MultipartSourceAdmission {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn is_initial(&self) -> bool {
|
||||
self.initial
|
||||
}
|
||||
pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
|
||||
let admission = self.clone();
|
||||
Arc::new(move || admission.validate())
|
||||
@@ -256,8 +250,6 @@ pub(super) struct MultipartCommitStaging {
|
||||
logical_path: PathBuf,
|
||||
io_path: PathBuf,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
cleaned: Arc<std::sync::atomic::AtomicBool>,
|
||||
fresh_source_head: Option<bytes::Bytes>,
|
||||
}
|
||||
|
||||
impl MultipartCommitStaging {
|
||||
@@ -267,19 +259,10 @@ impl MultipartCommitStaging {
|
||||
pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
|
||||
self.io_path.join(directory.to_string())
|
||||
}
|
||||
pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
|
||||
(self.io_path.clone(), Arc::clone(&self.cleaned))
|
||||
}
|
||||
pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
|
||||
self.fresh_source_head.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MultipartCommitStaging {
|
||||
fn drop(&mut self) {
|
||||
if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let logical_path = self.logical_path.clone();
|
||||
let io_path = self.io_path.clone();
|
||||
let owner = self.owner.clone();
|
||||
@@ -302,14 +285,6 @@ impl Drop for MultipartCommitStaging {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
let root = upload_scope_root(upload_or_part_path, part_path);
|
||||
let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
|
||||
return false;
|
||||
};
|
||||
super::multipart_source_state::has_clean_active_multipart_source(&source)
|
||||
}
|
||||
|
||||
pub(super) fn multipart_source_admission(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -381,14 +356,12 @@ impl LocalDisk {
|
||||
let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
|
||||
// The attempt root is exactly the namespace used by the tmp sweeper.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
|
||||
let mut staging = MultipartCommitStaging {
|
||||
let staging = MultipartCommitStaging {
|
||||
path: attempt,
|
||||
lease,
|
||||
logical_path,
|
||||
io_path,
|
||||
owner,
|
||||
cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
fresh_source_head: None,
|
||||
};
|
||||
let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
|
||||
let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
|
||||
@@ -398,37 +371,31 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
|
||||
let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
|
||||
(raw, source, true)
|
||||
} else {
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
(bytes::Bytes::from(raw), source, false)
|
||||
};
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let source_dir = source
|
||||
.data_dir
|
||||
.filter(|dir| !dir.is_nil())
|
||||
@@ -445,31 +412,23 @@ impl LocalDisk {
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let mut pinned_source_data = None;
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
pinned_source_data =
|
||||
Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
|
||||
}
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
|
||||
};
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
.max(actual.actual_size);
|
||||
@@ -505,15 +464,10 @@ impl LocalDisk {
|
||||
cancelled,
|
||||
)?;
|
||||
}
|
||||
let has_state = !from_active_cache
|
||||
&& (source_object.join(".mpu-source-state").exists()
|
||||
|| source_object.join(".mpu-recovery").exists()
|
||||
|| source_object.join(".mpu-completions").exists());
|
||||
Ok((!has_state).then_some(raw))
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
staging.fresh_source_head = fresh_head;
|
||||
Ok(staging)
|
||||
}
|
||||
}
|
||||
@@ -1265,211 +1219,4 @@ mod tests {
|
||||
.expect("non-part empty hash cleanup must remain supported");
|
||||
assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
|
||||
use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
|
||||
let (_root, disk, completed, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
|
||||
.await
|
||||
.expect("fresh multipart upload metadata");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
let read_fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("cached read_version");
|
||||
assert_eq!(read_fi.data_dir, Some(data_dir));
|
||||
|
||||
let destination = format!("{upload}/{data_dir}/part.1");
|
||||
let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part transaction");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"in-flight part transaction must mark active source as having unsettled part txn"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part transaction");
|
||||
assert!(
|
||||
disk.has_clean_active_multipart_source(&upload, false),
|
||||
"settled commit must restore clean active multipart source status"
|
||||
);
|
||||
let read_parts = disk
|
||||
.read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
|
||||
.await
|
||||
.expect("cached read_parts");
|
||||
assert_eq!(read_parts.len(), 1);
|
||||
assert_eq!(read_parts[0].number, 1);
|
||||
|
||||
// Prepare part 2, then replay Commit for already-settled part 1: part 2's
|
||||
// pending transaction must keep the upload marked as having unsettled part txns.
|
||||
let destination2 = format!("{upload}/{data_dir}/part.2");
|
||||
let mut part2_info = completed.parts[0].clone();
|
||||
part2_info.number = 2;
|
||||
let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part 2");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part 2 transaction");
|
||||
assert!(!disk.has_clean_active_multipart_source(&upload, false));
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replayed part 1 commit settlement");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"replayed settlement for part 1 must not clear part 2's pending transaction"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part 2");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part 2 transaction");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
// Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
|
||||
.await
|
||||
.expect("delete xl.meta via delete_paths");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"per-file head deletion must invalidate the canonical upload cache key"
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
|
||||
use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
|
||||
|
||||
let (_root, disk, _, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
let source_dir = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
|
||||
.expect("source dir");
|
||||
let base_dir = disk
|
||||
.io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.expect("multipart bucket path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
let init_wrapper = wrapper.clone();
|
||||
let init_upload = upload.clone();
|
||||
let init_task = tokio::spawn(async move {
|
||||
init_wrapper
|
||||
.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("initial write_metadata reaches parent dir fsync")
|
||||
.expect("entered channel");
|
||||
|
||||
let retire_wrapper = wrapper.clone();
|
||||
let retire_path = format!("{upload}/xl.meta");
|
||||
let mut retire_fut = Box::pin(async move {
|
||||
retire_wrapper
|
||||
.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
|
||||
"concurrent retirement must block on SourceRW while initial head publication is in flight"
|
||||
);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
init_task.await.expect("init join").expect("initial write_metadata succeeds");
|
||||
retire_fut
|
||||
.await
|
||||
.expect("concurrent retirement completes after SourceRW release");
|
||||
|
||||
assert!(
|
||||
!super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
|
||||
"retired source must not remain active in cache after interleaved initial publication"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
super::super::multipart_source_state::validate_source_mutation(
|
||||
&source_dir,
|
||||
&base_dir,
|
||||
&disk.publication_root,
|
||||
false
|
||||
),
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"validate_source_mutation must reject non-initial mutation on retired source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,13 +401,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
|
||||
Self::Remote(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+85
-411
@@ -1668,7 +1668,7 @@ fn clear_file_fdatasync_group_commit_for_test() {
|
||||
|
||||
// Small object directories are cheaper to flush in one blocking task. Multipart
|
||||
// directories fan out only once enough files can amortize per-task scheduling.
|
||||
pub(crate) const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16;
|
||||
const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16;
|
||||
pub(crate) const MAX_PARALLEL_FILE_SYNCS: usize = 16;
|
||||
// Scale aggregate fan-out for wider nodes while reserving at least half of the
|
||||
// configured Tokio blocking pool for unrelated filesystem work.
|
||||
@@ -1782,13 +1782,10 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
|
||||
limiter
|
||||
}
|
||||
|
||||
// Keep warm draining workers per disk/lane and spin briefly in user space
|
||||
// Keep one warm draining worker per disk/lane and spin briefly in user space
|
||||
// before exiting `spawn_blocking` so back-to-back stages of `PUT` and `DELETE`
|
||||
// on the same drive do not thrash Tokio's `BlockingPool` (`futex_wake`/`futex_wait`),
|
||||
// while allowing concurrent fsync callers to enter the kernel simultaneously and
|
||||
// coalesce in the filesystem journal (`jbd2`/`xfs` group commit).
|
||||
// on the same drive do not thrash Tokio's `BlockingPool` (`futex_wake`/`futex_wait`).
|
||||
const MAX_NAMESPACE_WORKERS_PER_DISK: usize = 1;
|
||||
pub(crate) const MAX_FSYNC_WORKERS_PER_DISK: usize = 8;
|
||||
// Return busy workers to the shared pool so other disks can acquire a slot.
|
||||
const MAX_BLOCKING_JOBS_PER_WORKER: usize = 64;
|
||||
const WORKER_IDLE_SPIN_ITERS: usize = 256;
|
||||
@@ -1816,7 +1813,6 @@ impl DiskBlockingLane {
|
||||
struct DiskBlockingQueueState {
|
||||
jobs: VecDeque<Box<dyn FnOnce() + Send>>,
|
||||
active_workers: usize,
|
||||
idle_workers: usize,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
paused_workers: usize,
|
||||
}
|
||||
@@ -1860,7 +1856,6 @@ impl Drop for DiskBlockingWorkerGuard {
|
||||
let cancelled = {
|
||||
let mut state = self.queue.lane(self.lane).state.lock();
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
state.idle_workers = state.idle_workers.saturating_sub(1);
|
||||
if state.active_workers == 0 {
|
||||
self.queue
|
||||
.lane(self.lane)
|
||||
@@ -1879,10 +1874,9 @@ impl Drop for DiskBlockingWorkerGuard {
|
||||
let should_respawn = {
|
||||
let mut state = self.queue.lane(self.lane).state.lock();
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
let max_workers = self.queue.effective_max_workers(self.lane, &state);
|
||||
if state.jobs.len() > state.idle_workers && state.active_workers < max_workers {
|
||||
let max_workers = self.queue.effective_max_workers(&state);
|
||||
if !state.jobs.is_empty() && state.active_workers < max_workers {
|
||||
state.active_workers += 1;
|
||||
state.idle_workers += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -1907,23 +1901,15 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn lane_max_workers(&self, lane: DiskBlockingLane) -> usize {
|
||||
match lane {
|
||||
DiskBlockingLane::Foreground | DiskBlockingLane::Isolated => MAX_NAMESPACE_WORKERS_PER_DISK,
|
||||
DiskBlockingLane::Fsync => MAX_FSYNC_WORKERS_PER_DISK,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn effective_max_workers(&self, lane: DiskBlockingLane, state: &DiskBlockingQueueState) -> usize {
|
||||
fn effective_max_workers(&self, state: &DiskBlockingQueueState) -> usize {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
self.lane_max_workers(lane).saturating_add(state.paused_workers)
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK.saturating_add(state.paused_workers)
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
{
|
||||
let _ = state;
|
||||
self.lane_max_workers(lane)
|
||||
MAX_NAMESPACE_WORKERS_PER_DISK
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1932,9 +1918,8 @@ impl DiskBlockingQueue {
|
||||
let mut state = self.lane(lane).state.lock();
|
||||
state.jobs.push_back(job);
|
||||
self.lane(lane).ready.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
if state.jobs.len() > state.idle_workers && state.active_workers < self.effective_max_workers(lane, &state) {
|
||||
if state.active_workers < self.effective_max_workers(&state) {
|
||||
state.active_workers += 1;
|
||||
state.idle_workers += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -1977,19 +1962,13 @@ impl DiskBlockingQueue {
|
||||
CURRENT_DISK_BLOCKING_QUEUE.with(|slot| {
|
||||
*slot.borrow_mut() = Some((Arc::clone(&queue), lane));
|
||||
});
|
||||
guard.state = DiskBlockingWorkerState::Running;
|
||||
let mut remaining_jobs = MAX_BLOCKING_JOBS_PER_WORKER;
|
||||
let mut idle_polled = false;
|
||||
let mut counted_idle = true;
|
||||
loop {
|
||||
let job = {
|
||||
let mut state = queue.lane(lane).state.lock();
|
||||
if counted_idle {
|
||||
state.idle_workers = state.idle_workers.saturating_sub(1);
|
||||
counted_idle = false;
|
||||
}
|
||||
guard.state = DiskBlockingWorkerState::Running;
|
||||
if state.active_workers > queue.effective_max_workers(lane, &state) || (idle_polled && state.jobs.is_empty())
|
||||
{
|
||||
if state.active_workers > queue.effective_max_workers(&state) || (idle_polled && state.jobs.is_empty()) {
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
guard.state = DiskBlockingWorkerState::Retired;
|
||||
return;
|
||||
@@ -1999,15 +1978,6 @@ impl DiskBlockingQueue {
|
||||
.lane(lane)
|
||||
.ready
|
||||
.store(!state.jobs.is_empty(), std::sync::atomic::Ordering::Relaxed);
|
||||
if job.is_none() {
|
||||
if state.idle_workers > 0 {
|
||||
state.active_workers = state.active_workers.saturating_sub(1);
|
||||
guard.state = DiskBlockingWorkerState::Retired;
|
||||
return;
|
||||
}
|
||||
state.idle_workers += 1;
|
||||
counted_idle = true;
|
||||
}
|
||||
job
|
||||
};
|
||||
if let Some(job) = job {
|
||||
@@ -2018,11 +1988,15 @@ impl DiskBlockingQueue {
|
||||
}
|
||||
idle_polled = false;
|
||||
} else {
|
||||
for _ in 0..WORKER_IDLE_SPIN_ITERS {
|
||||
for i in 0..WORKER_IDLE_SPIN_ITERS {
|
||||
if queue.lane(lane).ready.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
std::hint::spin_loop();
|
||||
if i & 15 == 15 {
|
||||
std::thread::yield_now();
|
||||
} else {
|
||||
std::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
// Recheck and retire under the same mutex used by enqueue.
|
||||
idle_polled = true;
|
||||
@@ -2132,9 +2106,8 @@ pub(crate) fn enter_blocking_test_hook() -> BlockingTestHookGuard {
|
||||
let should_spawn = {
|
||||
let mut state = queue.lane(lane).state.lock();
|
||||
state.paused_workers = state.paused_workers.saturating_add(1);
|
||||
if state.jobs.len() > state.idle_workers && state.active_workers < queue.effective_max_workers(lane, &state) {
|
||||
if !state.jobs.is_empty() && state.active_workers < queue.effective_max_workers(&state) {
|
||||
state.active_workers += 1;
|
||||
state.idle_workers += 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -2596,7 +2569,7 @@ fn sync_files(paths: &[PathBuf]) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
let mut files = Vec::with_capacity(PARALLEL_FILE_SYNC_THRESHOLD);
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
@@ -2779,114 +2752,22 @@ pub(crate) fn read_dir_probe(path: impl AsRef<Path>, entry_limit: usize) -> io::
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn has_non_data_dir_entry(path: impl AsRef<Path>, data_dirs: &[uuid::Uuid]) -> io::Result<bool> {
|
||||
has_non_data_dir_entry_lazy(path, || data_dirs.to_vec()).map(|(_, res)| res)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn has_non_data_dir_entry_lazy_from_fd(
|
||||
dir_fd: &std::fs::File,
|
||||
load_data_dirs: impl FnOnce() -> Vec<uuid::Uuid>,
|
||||
) -> io::Result<(Vec<uuid::Uuid>, bool)> {
|
||||
use rustix::fs::{AtFlags, FileType, RawDir, statat};
|
||||
|
||||
let mut raw_buf = [std::mem::MaybeUninit::<u8>::uninit(); 1024];
|
||||
let mut dir = RawDir::new(dir_fd, &mut raw_buf);
|
||||
let mut data_dirs: Option<Vec<uuid::Uuid>> = None;
|
||||
let mut load = Some(load_data_dirs);
|
||||
let mut valid_entries = 0usize;
|
||||
|
||||
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
|
||||
let Some(entry) = dir.next() else {
|
||||
return Ok((data_dirs.unwrap_or_default(), false));
|
||||
};
|
||||
let entry = match entry {
|
||||
Ok(entry) => entry,
|
||||
Err(rustix::io::Errno::NOENT) => return Ok((data_dirs.unwrap_or_default(), false)),
|
||||
Err(err) => return Err(io::Error::from(err)),
|
||||
};
|
||||
let c_name = entry.file_name();
|
||||
let name_bytes = c_name.to_bytes();
|
||||
if name_bytes.is_empty() || name_bytes == b"." || name_bytes == b".." {
|
||||
continue;
|
||||
}
|
||||
|
||||
let file_type = match entry.file_type() {
|
||||
FileType::Unknown => match statat(dir_fd, c_name, AtFlags::SYMLINK_NOFOLLOW) {
|
||||
Ok(st) => FileType::from_raw_mode(st.st_mode),
|
||||
Err(rustix::io::Errno::NOENT) => continue,
|
||||
Err(err) => return Err(io::Error::from(err)),
|
||||
},
|
||||
ft => ft,
|
||||
};
|
||||
if file_type == FileType::Directory {
|
||||
let parsed_uuid = if name_bytes.len() == uuid::fmt::Hyphenated::LENGTH {
|
||||
std::str::from_utf8(name_bytes).ok().and_then(|s| {
|
||||
let parsed = uuid::Uuid::try_parse(s).ok()?;
|
||||
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
|
||||
(parsed.hyphenated().encode_lower(&mut buf) == s).then_some(parsed)
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let Some(parsed) = parsed_uuid else {
|
||||
let dirs = data_dirs
|
||||
.take()
|
||||
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
|
||||
return Ok((dirs, true));
|
||||
};
|
||||
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
|
||||
if dirs.binary_search(&parsed).is_err() {
|
||||
return Ok((std::mem::take(dirs), true));
|
||||
}
|
||||
valid_entries += 1;
|
||||
} else if file_type == FileType::RegularFile {
|
||||
valid_entries += 1;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
|
||||
let entry_limit = data_dirs.as_ref().map_or(2, |dirs| dirs.len().saturating_add(2));
|
||||
if valid_entries == entry_limit {
|
||||
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
|
||||
if valid_entries == dirs.len().saturating_add(2) {
|
||||
return Ok((std::mem::take(dirs), true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let dirs = data_dirs
|
||||
.take()
|
||||
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
|
||||
Ok((dirs, true))
|
||||
}
|
||||
|
||||
#[cfg(any(not(target_os = "linux"), test))]
|
||||
pub(crate) fn has_non_data_dir_entry_lazy(
|
||||
path: impl AsRef<Path>,
|
||||
load_data_dirs: impl FnOnce() -> Vec<uuid::Uuid>,
|
||||
) -> io::Result<(Vec<uuid::Uuid>, bool)> {
|
||||
match has_non_data_dir_entry_lazy_inner(path.as_ref(), load_data_dirs) {
|
||||
match has_non_data_dir_entry_inner(path.as_ref(), data_dirs) {
|
||||
Ok(res) => Ok(res),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok((Vec::new(), false)),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(not(target_os = "linux"), test))]
|
||||
fn has_non_data_dir_entry_lazy_inner(
|
||||
path: &Path,
|
||||
load_data_dirs: impl FnOnce() -> Vec<uuid::Uuid>,
|
||||
) -> io::Result<(Vec<uuid::Uuid>, bool)> {
|
||||
fn has_non_data_dir_entry_inner(path: &Path, data_dirs: &[uuid::Uuid]) -> io::Result<bool> {
|
||||
let mut dir = std::fs::read_dir(path)?;
|
||||
let mut data_dirs: Option<Vec<uuid::Uuid>> = None;
|
||||
let mut load = Some(load_data_dirs);
|
||||
let entry_limit = data_dirs.len().saturating_add(2);
|
||||
let mut valid_entries = 0usize;
|
||||
|
||||
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
|
||||
let Some(entry) = dir.next() else {
|
||||
return Ok((data_dirs.unwrap_or_default(), false));
|
||||
return Ok(false);
|
||||
};
|
||||
let entry = entry?;
|
||||
let file_name = entry.file_name();
|
||||
@@ -2896,24 +2777,18 @@ fn has_non_data_dir_entry_lazy_inner(
|
||||
|
||||
let file_type = entry.file_type()?;
|
||||
if file_type.is_dir() {
|
||||
let parsed_uuid = if file_name.len() == uuid::fmt::Hyphenated::LENGTH {
|
||||
file_name.to_str().and_then(|s| {
|
||||
let parsed = uuid::Uuid::try_parse(s).ok()?;
|
||||
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
|
||||
(parsed.hyphenated().encode_lower(&mut buf) == s).then_some(parsed)
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let Some(parsed) = parsed_uuid else {
|
||||
let dirs = data_dirs
|
||||
.take()
|
||||
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
|
||||
return Ok((dirs, true));
|
||||
};
|
||||
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
|
||||
if dirs.binary_search(&parsed).is_err() {
|
||||
return Ok((std::mem::take(dirs), true));
|
||||
let is_data_dir = file_name.len() == uuid::fmt::Hyphenated::LENGTH
|
||||
&& file_name
|
||||
.to_str()
|
||||
.and_then(|s| {
|
||||
let parsed = uuid::Uuid::try_parse(s).ok()?;
|
||||
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
|
||||
(parsed.hyphenated().encode_lower(&mut buf) == s && data_dirs.binary_search(&parsed).is_ok())
|
||||
.then_some(())
|
||||
})
|
||||
.is_some();
|
||||
if !is_data_dir {
|
||||
return Ok(true);
|
||||
}
|
||||
valid_entries += 1;
|
||||
} else if file_type.is_file() {
|
||||
@@ -2922,19 +2797,12 @@ fn has_non_data_dir_entry_lazy_inner(
|
||||
continue;
|
||||
}
|
||||
|
||||
let entry_limit = data_dirs.as_ref().map_or(2, |dirs| dirs.len().saturating_add(2));
|
||||
if valid_entries == entry_limit {
|
||||
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
|
||||
if valid_entries == dirs.len().saturating_add(2) {
|
||||
return Ok((std::mem::take(dirs), true));
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
let dirs = data_dirs
|
||||
.take()
|
||||
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
|
||||
Ok((dirs, true))
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
// read_dir count read limit. when count == 0 unlimit.
|
||||
@@ -3769,21 +3637,16 @@ impl PreparedRenameSource {
|
||||
#[cfg(not(windows))]
|
||||
std::io::Write::write_all(&mut self.source, data)?;
|
||||
if sync {
|
||||
self.sync_data()?;
|
||||
#[cfg(test)]
|
||||
multipart_source_syscall_failures::check(multipart_source_syscall_failures::Stage::FileSync, &self.test_destination)?;
|
||||
let _sync = StageTimer::new(true, "disk_prepared_metadata_fdatasync");
|
||||
#[cfg(windows)]
|
||||
self.source.as_file().sync_data()?;
|
||||
#[cfg(not(windows))]
|
||||
self.source.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn sync_data(&self) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
multipart_source_syscall_failures::check(multipart_source_syscall_failures::Stage::FileSync, &self.test_destination)?;
|
||||
let _sync = StageTimer::new(true, "disk_prepared_metadata_fdatasync");
|
||||
#[cfg(windows)]
|
||||
self.source.as_file().sync_data()?;
|
||||
#[cfg(not(windows))]
|
||||
self.source.sync_data()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn create_prepared_rename_source_with_commit_guard(
|
||||
@@ -4969,17 +4832,6 @@ impl PublicationRoot {
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn open_directory_and_readonly_child(
|
||||
&self,
|
||||
dir_path: &Path,
|
||||
child_name: &str,
|
||||
) -> io::Result<(std::fs::File, std::fs::File)> {
|
||||
let dir = self.open_directory(dir_path)?;
|
||||
let child = open_linux_relative(&dir, Path::new(child_name), rustix::fs::OFlags::RDONLY, rustix::fs::Mode::empty())?;
|
||||
Ok((dir, child))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn relative_path<'a>(&self, path: &'a Path) -> io::Result<&'a Path> {
|
||||
path.strip_prefix(self.io_path.as_path())
|
||||
@@ -5445,34 +5297,6 @@ impl RenameCommitGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn multipart_payload_commit_guard_std(
|
||||
metadata_guard: &RenameCommitGuard,
|
||||
source_parent: &Path,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let original = &metadata_guard.directories;
|
||||
original.validate()?;
|
||||
let source = LinuxCommitDirectory::new(original.root.open_directory(source_parent)?)?;
|
||||
let destination = original.prepare_destination()?;
|
||||
// Clone the pinned file identity, never recapture the target pathname
|
||||
// between payload and canonical-head publication.
|
||||
let destination = LinuxCommitDirectory {
|
||||
file: Arc::clone(&destination.file),
|
||||
device: destination.device,
|
||||
inode: destination.inode,
|
||||
};
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: original.root.clone(),
|
||||
source_path: source_parent.to_path_buf(),
|
||||
destination_path: original.destination_path.clone(),
|
||||
source,
|
||||
destination: std::sync::OnceLock::from(destination),
|
||||
destination_base: original.destination_base.clone(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Native MPU metadata and payload have distinct source parents. Reuse the
|
||||
/// exact target identity already held by the metadata transaction, while
|
||||
/// independently pinning the private payload source under the same root.
|
||||
@@ -5484,7 +5308,30 @@ pub(crate) async fn multipart_payload_commit_guard(
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let metadata_guard = metadata_guard.clone();
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
run_blocking_namespace_operation(lease, move || multipart_payload_commit_guard_std(&metadata_guard, &source_parent)).await
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let original = &metadata_guard.directories;
|
||||
original.validate()?;
|
||||
let source = LinuxCommitDirectory::new(original.root.open_directory(&source_parent)?)?;
|
||||
let destination = original.prepare_destination()?;
|
||||
// Clone the pinned file identity, never recapture the target pathname
|
||||
// between payload and canonical-head publication.
|
||||
let destination = LinuxCommitDirectory {
|
||||
file: Arc::clone(&destination.file),
|
||||
device: destination.device,
|
||||
inode: destination.inode,
|
||||
};
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
root: original.root.clone(),
|
||||
source_path: source_parent,
|
||||
destination_path: original.destination_path.clone(),
|
||||
source,
|
||||
destination: std::sync::OnceLock::from(destination),
|
||||
destination_base: original.destination_base.clone(),
|
||||
}),
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_rename_commit_guard(
|
||||
@@ -6666,17 +6513,6 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
include_leaf: bool,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
sync_multipart_owned_record_parents_options(source, record_parent, include_leaf, true, base, publication_root)
|
||||
}
|
||||
|
||||
pub(crate) fn sync_multipart_owned_record_parents_options(
|
||||
source: &Path,
|
||||
record_parent: &Path,
|
||||
include_leaf: bool,
|
||||
sync_source_root: bool,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
let relative = record_parent
|
||||
.strip_prefix(source)
|
||||
@@ -6700,9 +6536,6 @@ pub(crate) fn sync_multipart_owned_record_parents_options(
|
||||
if !directory.starts_with(source) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
|
||||
}
|
||||
if directory == source && !sync_source_root {
|
||||
return Ok(());
|
||||
}
|
||||
#[cfg_attr(not(unix), allow(unused_mut))]
|
||||
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
#[cfg(unix)]
|
||||
@@ -11152,54 +10985,6 @@ mod tests {
|
||||
assert_eq!(err.kind(), io::ErrorKind::NotFound);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_non_data_dir_entry_lazy_skips_loader_when_only_xl_meta_exists() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
std::fs::write(temp_dir.path().join("xl.meta"), b"meta").expect("write xl.meta");
|
||||
let mut called = 0usize;
|
||||
let (dirs, has_child) = has_non_data_dir_entry_lazy(temp_dir.path(), || {
|
||||
called += 1;
|
||||
vec![uuid::Uuid::new_v4()]
|
||||
})
|
||||
.expect("lazy check must succeed");
|
||||
assert_eq!(called, 0, "inline object dir without subdirs must not invoke loader");
|
||||
assert!(dirs.is_empty());
|
||||
assert!(!has_child);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let pub_root = PublicationRoot::new(temp_dir.path()).expect("open pub root");
|
||||
let (dir_fd, _meta) = pub_root
|
||||
.open_directory_and_readonly_child(temp_dir.path(), "xl.meta")
|
||||
.expect("open dir and child");
|
||||
let mut fd_called = 0usize;
|
||||
let (fd_dirs, fd_has_child) = has_non_data_dir_entry_lazy_from_fd(&dir_fd, || {
|
||||
fd_called += 1;
|
||||
vec![uuid::Uuid::new_v4()]
|
||||
})
|
||||
.expect("fd lazy check must succeed");
|
||||
assert_eq!(fd_called, 0);
|
||||
assert!(fd_dirs.is_empty());
|
||||
assert!(!fd_has_child);
|
||||
}
|
||||
|
||||
let data_dir = uuid::Uuid::new_v4();
|
||||
std::fs::create_dir(temp_dir.path().join(data_dir.to_string())).expect("create data dir");
|
||||
let (dirs, has_child) = has_non_data_dir_entry_lazy(temp_dir.path(), || {
|
||||
called += 1;
|
||||
vec![data_dir]
|
||||
})
|
||||
.expect("lazy check with data dir must succeed");
|
||||
assert_eq!(called, 1, "data dir presence must invoke loader once");
|
||||
assert_eq!(dirs, vec![data_dir]);
|
||||
assert!(!has_child);
|
||||
|
||||
std::fs::create_dir(temp_dir.path().join("nested-key")).expect("create nested child dir");
|
||||
let (dirs, has_child) = has_non_data_dir_entry_lazy(temp_dir.path(), || vec![data_dir]).expect("lazy check child");
|
||||
assert_eq!(dirs, vec![data_dir], "child presence must still return data_dirs for skip set");
|
||||
assert!(has_child);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn resource_grouped_fdatasync_retired_limiter_keeps_its_budget_until_worker_cleanup() {
|
||||
@@ -11443,24 +11228,19 @@ mod tests {
|
||||
let queue = register_disk_blocking_queue(root.path(), root.path());
|
||||
std::fs::write(root.path().join("intermediate"), b"backup").expect("backup");
|
||||
std::fs::write(root.path().join("receipt"), b"committed").expect("receipt");
|
||||
let mut release_txs = Vec::new();
|
||||
let mut blockers = Vec::new();
|
||||
for _ in 0..MAX_FSYNC_WORKERS_PER_DISK {
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let blocker_queue = Arc::clone(&queue);
|
||||
blockers.push(tokio::spawn(async move {
|
||||
blocker_queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}));
|
||||
started_rx.await.expect("fsync worker blocked");
|
||||
release_txs.push(release_tx);
|
||||
}
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let blocker_queue = Arc::clone(&queue);
|
||||
let blocker = tokio::spawn(async move {
|
||||
blocker_queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.expect("fsync lane blocked");
|
||||
let mut cleanup = std::pin::pin!(remove_files_and_fsync_with_owner(
|
||||
root.path(),
|
||||
&["intermediate"],
|
||||
@@ -11471,12 +11251,8 @@ mod tests {
|
||||
assert!(futures::FutureExt::now_or_never(&mut cleanup).is_none());
|
||||
let foreground = tokio::time::timeout(Duration::from_secs(5), queue.run(|| Ok(()))).await;
|
||||
let retained = root.path().join("receipt").exists();
|
||||
for release_tx in release_txs {
|
||||
release_tx.send(()).expect("release fsync worker");
|
||||
}
|
||||
for blocker in blockers {
|
||||
blocker.await.expect("blocker joined").expect("blocker completed");
|
||||
}
|
||||
release_tx.send(()).expect("release fsync lane");
|
||||
blocker.await.expect("blocker joined").expect("blocker completed");
|
||||
cleanup.await.expect("cleanup completed");
|
||||
foreground.expect("foreground must progress").expect("foreground operation");
|
||||
assert_eq!(retained, sync, "only durable cleanup must wait for the fsync lane");
|
||||
@@ -11698,25 +11474,6 @@ mod tests {
|
||||
|
||||
for lane in [DiskBlockingLane::Foreground, DiskBlockingLane::Fsync] {
|
||||
let queue = DiskBlockingQueue::new();
|
||||
let max_workers = queue.lane_max_workers(lane);
|
||||
let mut hold_txs = Vec::new();
|
||||
let mut hold_tasks = Vec::new();
|
||||
for _ in 1..max_workers {
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let hold_queue = Arc::clone(&queue);
|
||||
hold_tasks.push(tokio::spawn(async move {
|
||||
hold_queue
|
||||
.run_on_lane(lane, "hold operation failed", DiskBlockingQueue::test_stages_for_lane(lane), move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}));
|
||||
started_rx.await.expect("hold worker started");
|
||||
hold_txs.push(release_tx);
|
||||
}
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
|
||||
let (first_started_tx, first_started_rx) = oneshot::channel();
|
||||
let (first_release_tx, first_release_rx) = oneshot::channel();
|
||||
@@ -11750,91 +11507,10 @@ mod tests {
|
||||
let retained = gate.snapshot().active;
|
||||
second_release_tx.send(()).expect("release second producer");
|
||||
second.await.expect("second producer completed");
|
||||
for tx in hold_txs {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
for task in hold_tasks {
|
||||
task.await.expect("hold task joined").expect("hold task completed");
|
||||
}
|
||||
assert_eq!(retained, 0, "shared worker must not retain a completed producer's unit");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn disk_blocking_queue_fsync_lane_scales_workers_and_retires_idle() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let queue = DiskBlockingQueue::new();
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let max_seen = Arc::new(AtomicUsize::new(0));
|
||||
let entered = Arc::new(AtomicUsize::new(0));
|
||||
let (all_entered_tx, all_entered_rx) = oneshot::channel();
|
||||
let all_entered_tx = Arc::new(Mutex::new(Some(all_entered_tx)));
|
||||
let (release_tx, _) = tokio::sync::broadcast::channel::<()>(1);
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..(MAX_FSYNC_WORKERS_PER_DISK + 4) {
|
||||
let q = Arc::clone(&queue);
|
||||
let active = Arc::clone(&active);
|
||||
let max_seen = Arc::clone(&max_seen);
|
||||
let entered = Arc::clone(&entered);
|
||||
let all_entered_tx = Arc::clone(&all_entered_tx);
|
||||
let mut release_rx = release_tx.subscribe();
|
||||
handles.push(tokio::spawn(async move {
|
||||
use_current_runtime_for_fsync();
|
||||
q.run_on_lane(
|
||||
DiskBlockingLane::Fsync,
|
||||
"fsync scale test failed",
|
||||
DiskBlockingQueue::test_stages_for_lane(DiskBlockingLane::Fsync),
|
||||
move |_| {
|
||||
let cur = active.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
max_seen.fetch_max(cur, Ordering::SeqCst);
|
||||
let ord = entered.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if ord == MAX_FSYNC_WORKERS_PER_DISK
|
||||
&& let Some(tx) = all_entered_tx.lock().expect("lock completion sender").take()
|
||||
{
|
||||
let _ = tx.send(());
|
||||
}
|
||||
if ord <= MAX_FSYNC_WORKERS_PER_DISK {
|
||||
let _ = release_rx.blocking_recv();
|
||||
}
|
||||
active.fetch_sub(1, Ordering::SeqCst);
|
||||
Ok(i)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), all_entered_rx)
|
||||
.await
|
||||
.expect("all fsync workers should enter within timeout")
|
||||
.expect("all_entered channel");
|
||||
let _ = release_tx.send(());
|
||||
for (i, handle) in handles.into_iter().enumerate() {
|
||||
assert_eq!(handle.await.expect("task join").expect("fsync job"), i);
|
||||
}
|
||||
assert_eq!(
|
||||
max_seen.load(Ordering::SeqCst),
|
||||
MAX_FSYNC_WORKERS_PER_DISK,
|
||||
"fsync lane must run MAX_FSYNC_WORKERS_PER_DISK concurrent workers up to its configured bound"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
{
|
||||
let state = queue.lane(DiskBlockingLane::Fsync).state.lock();
|
||||
if state.active_workers == 0 && state.idle_workers == 0 {
|
||||
assert!(state.jobs.is_empty());
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all fsync workers must retire with zero idle count");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn disk_blocking_queue_bounds_concurrency_and_recovers_from_panic() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -11842,7 +11518,6 @@ mod tests {
|
||||
let queue = DiskBlockingQueue::new();
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let max_seen = Arc::new(AtomicUsize::new(0));
|
||||
let entered = Arc::new(AtomicUsize::new(0));
|
||||
let barrier = Arc::new(std::sync::Barrier::new(MAX_NAMESPACE_WORKERS_PER_DISK + 1));
|
||||
|
||||
let mut handles = Vec::new();
|
||||
@@ -11850,13 +11525,12 @@ mod tests {
|
||||
let q = Arc::clone(&queue);
|
||||
let active = Arc::clone(&active);
|
||||
let max_seen = Arc::clone(&max_seen);
|
||||
let entered = Arc::clone(&entered);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
handles.push(tokio::spawn(async move {
|
||||
q.run(move || {
|
||||
let cur = active.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
max_seen.fetch_max(cur, Ordering::SeqCst);
|
||||
if entered.fetch_add(1, Ordering::SeqCst) < MAX_NAMESPACE_WORKERS_PER_DISK {
|
||||
if i < MAX_NAMESPACE_WORKERS_PER_DISK {
|
||||
barrier.wait();
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
|
||||
@@ -828,27 +828,13 @@ pub async fn create_bitrot_writer(
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
create_bitrot_writer_with_orig_volume(is_inline_buffer, disk, "", volume, path, length, shard_size, checksum_algo).await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn create_bitrot_writer_with_orig_volume(
|
||||
is_inline_buffer: bool,
|
||||
disk: Option<&DiskStore>,
|
||||
origvolume: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
let writer = if is_inline_buffer {
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else if let Some(disk) = disk {
|
||||
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
|
||||
|
||||
let file = disk.create_file(origvolume, volume, path, length).await?;
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let file = instrument_raw_shard_writer(file, disk.is_local());
|
||||
CustomWriter::new_tokio_writer(file)
|
||||
|
||||
@@ -333,34 +333,6 @@ struct ProofCache {
|
||||
preferred: usize,
|
||||
}
|
||||
|
||||
type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
|
||||
|
||||
static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.get(&(part.generation, part.root, stripe))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
|
||||
let mut map = VERIFIED_STRIPE_RECORDS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().copied()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert((part.generation, part.root, stripe), record);
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.retain(|(stored_gen, _, _), _| *stored_gen != generation);
|
||||
}
|
||||
|
||||
impl PartProofReader {
|
||||
pub(crate) async fn reconstruction_proof(
|
||||
&self,
|
||||
@@ -438,9 +410,6 @@ impl PartProofReader {
|
||||
|
||||
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
|
||||
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
|
||||
if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let mut cached = self.cached.lock().await;
|
||||
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
|
||||
return Ok(bytes.clone());
|
||||
@@ -496,9 +465,6 @@ impl PartProofReader {
|
||||
bytes
|
||||
};
|
||||
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
if self.part.stripes <= 4 {
|
||||
put_verified_stripe_record(&self.part, stripe, record.clone());
|
||||
}
|
||||
if cached.records.len() == 2 {
|
||||
cached.records.pop_front();
|
||||
}
|
||||
@@ -897,41 +863,4 @@ mod tests {
|
||||
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
|
||||
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let inline_bytes = prepared.inline_bytes().expect("inline bytes");
|
||||
let part = prepared.part;
|
||||
|
||||
let warm_reader = PartProofReader {
|
||||
part: part.clone(),
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: Some(inline_bytes),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
|
||||
assert!(
|
||||
get_verified_stripe_record(&part, 0).is_some(),
|
||||
"small verified stripe record should populate the verified stripe cache"
|
||||
);
|
||||
|
||||
let cached_reader = Arc::new(PartProofReader {
|
||||
part,
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
|
||||
assert!(
|
||||
cached_reader.verify(0, 0, shards[1]).await.is_err(),
|
||||
"cached record must still reject corrupt or mismatched shard payloads"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1543,8 +1543,6 @@ pub struct ObjectInfo {
|
||||
/// published by the same upload. Callers must not repeat quota, replication,
|
||||
/// lifecycle, or object-created side effects for that response.
|
||||
pub multipart_completion_replayed: bool,
|
||||
/// Previous current version's size observed during `rename_data` commit.
|
||||
pub backfilled_old_current_size: Option<crate::disk::OldCurrentSize>,
|
||||
}
|
||||
|
||||
impl Default for ObjectInfo {
|
||||
@@ -1587,7 +1585,6 @@ impl Default for ObjectInfo {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1631,7 +1628,6 @@ impl Clone for ObjectInfo {
|
||||
replication_decision: self.replication_decision.clone(),
|
||||
checksum: self.checksum.clone(),
|
||||
multipart_completion_replayed: self.multipart_completion_replayed,
|
||||
backfilled_old_current_size: self.backfilled_old_current_size,
|
||||
expires: self.expires,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5340,14 +5340,6 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
|
||||
@@ -5547,14 +5539,6 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
@@ -5614,11 +5598,10 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
let part = ObjectPartInfo::unmarshal(&meta)?;
|
||||
let integrity = part.integrity.map(Arc::new);
|
||||
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
@@ -5640,9 +5623,7 @@ impl SetDisks {
|
||||
if let Err(error) = prepared {
|
||||
return Some(Err(error));
|
||||
}
|
||||
if let Some(integrity) = integrity
|
||||
&& !disk.is_local()
|
||||
{
|
||||
if let Some(integrity) = integrity {
|
||||
let Some((directory, _)) = dst_object.rsplit_once('/') else {
|
||||
return Some(Err(DiskError::FileCorrupt));
|
||||
};
|
||||
@@ -5759,36 +5740,7 @@ impl SetDisks {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let all_prepared_succeeded = errs
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
|
||||
let committed = if all_prepared_succeeded {
|
||||
let settles = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Commit)
|
||||
.await
|
||||
}
|
||||
});
|
||||
let settle_results = join_all(settles).await;
|
||||
if settle_results.iter().filter(|res| res.is_ok()).count() >= write_quorum
|
||||
&& settle_results
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(res, disk)| disk.is_none() || res.is_ok())
|
||||
{
|
||||
true
|
||||
} else {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?
|
||||
}
|
||||
} else {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?
|
||||
};
|
||||
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
if !committed {
|
||||
let err = DiskError::ErasureWriteQuorum;
|
||||
if let Some(context) = quorum_context {
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for (version_id, parts) in versions {
|
||||
for parts in versions.into_values() {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,25 +173,15 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
|
||||
@@ -80,7 +80,7 @@ use crate::erasure::codec::bridge::{
|
||||
use crate::erasure::coding;
|
||||
use crate::error::{Error, Result, is_err_version_not_found};
|
||||
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer, create_bitrot_writer_with_orig_volume};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::get_object_body_cache_hook;
|
||||
use crate::object_api::object_api_utils::get_raw_etag;
|
||||
@@ -9038,13 +9038,6 @@ mod tests {
|
||||
"constructing an object lock must not clone each client Arc"
|
||||
);
|
||||
|
||||
// Check shared ownership before acquisition can spawn pending cleanup.
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
|
||||
let write_error = write_lock
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9059,6 +9052,11 @@ mod tests {
|
||||
),
|
||||
"the shared client representation must preserve the exact write quorum result: {write_error}"
|
||||
);
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
let read_guard = read_lock
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
|
||||
@@ -41,8 +41,8 @@ use super::super::{
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
|
||||
create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
|
||||
get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
|
||||
get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
|
||||
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
@@ -124,10 +124,9 @@ async fn create_part_writers(
|
||||
let Some(disk) = disk else {
|
||||
return (None, Some(DiskError::DiskNotFound));
|
||||
};
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
match create_bitrot_writer(
|
||||
false,
|
||||
Some(disk),
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
path,
|
||||
length,
|
||||
@@ -978,7 +977,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
if !root_prefix.is_empty() {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1003,13 +1002,6 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.into_iter()
|
||||
.map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
|
||||
.filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.map(|sha_dir| {
|
||||
if root_prefix.is_empty() {
|
||||
sha_dir
|
||||
} else {
|
||||
format!("{root_prefix}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let listed_upload_dirs = stream::iter(sha_dirs)
|
||||
.map(|sha_dir| {
|
||||
@@ -1132,7 +1124,6 @@ impl SetDisks {
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -1174,8 +1165,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
|
||||
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
|
||||
}
|
||||
|
||||
@@ -1197,7 +1187,7 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
@@ -1210,9 +1200,8 @@ impl SetDisks {
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _, has_minority_candidate) = self
|
||||
.discover_multipart_upload_paths(bucket, object, &expected_parent, false)
|
||||
.await?;
|
||||
let (_, candidate_paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::DecommissionCapacityBlocked {
|
||||
message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
|
||||
@@ -1761,54 +1750,6 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
// Every disk must confirm absence in both namespaces before cleanup can
|
||||
// remove catalog protection from the bucket.
|
||||
for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let (disks, paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for upload_path in paths {
|
||||
let (sha_dir, _) = upload_path
|
||||
.rsplit_once('/')
|
||||
.filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
|
||||
.ok_or(DiskError::CorruptedFormat)?;
|
||||
let (metadata, errors) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
|
||||
.await?;
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
|
||||
let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
|
||||
let (Some(owner_bucket), Some(owner_object)) = (
|
||||
file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) else {
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
};
|
||||
if owner_bucket.is_empty()
|
||||
|| owner_object.is_empty()
|
||||
|| Self::get_multipart_sha_dir(owner_bucket, owner_object)
|
||||
!= sha_dir
|
||||
.strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.unwrap_or(sha_dir)
|
||||
{
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
if owner_bucket == bucket {
|
||||
return Err(Error::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1820,8 +1761,7 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -2496,16 +2436,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
let part_integrity = if let Some(integrity) = integrity {
|
||||
Some(
|
||||
integrity
|
||||
.write(
|
||||
&mut shuffle_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
|
||||
} else { None };
|
||||
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
@@ -3540,6 +3471,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
bind_multipart_proof_snapshot(&disks, &self.multipart_proof_verification_context())?;
|
||||
|
||||
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let part_meta_paths = uploaded_parts
|
||||
.iter()
|
||||
.map(|v| format!("{part_path}part.{0}.meta", v.part_num))
|
||||
@@ -3547,10 +3482,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
|
||||
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let object_parts = Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
@@ -4458,7 +4389,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let needs_immediate_heal = rename_commit.needs_immediate_heal();
|
||||
let op_old_dir = rename_commit.data_dir;
|
||||
let cleanup_disks = rename_commit.cleanup_disks;
|
||||
let old_current_size = rename_commit.old_current_size;
|
||||
let committed_file_info = rename_commit.committed_file_info;
|
||||
|
||||
if needs_immediate_heal {
|
||||
@@ -4527,13 +4457,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
let can_skip_part_cleanup = !needs_upgrade
|
||||
&& commit_disks
|
||||
.iter()
|
||||
.all(|disk| disk.as_ref().is_some_and(|disk| disk.is_local()));
|
||||
if !can_skip_part_cleanup {
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
}
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
@@ -4579,9 +4503,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
drop(_upload_guard.take());
|
||||
|
||||
let mut obj_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned);
|
||||
obj_info.backfilled_old_current_size = old_current_size;
|
||||
Ok(obj_info)
|
||||
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
|
||||
};
|
||||
|
||||
let result = if detach_commit_owner {
|
||||
@@ -11272,163 +11194,6 @@ mod tests {
|
||||
assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-target";
|
||||
let other = "multipart-cleanup-other";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create target bucket");
|
||||
disk.make_volume(other).await.expect("create other bucket");
|
||||
}
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an empty upload namespace permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(other, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in other bucket");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an upload owned by another bucket permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in target bucket");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-offline";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks.disks.write().await[3] = None;
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing preserves its discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the offline disk must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-list-error";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
|
||||
let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
|
||||
tokio::fs::create_dir_all(&multipart_dir)
|
||||
.await
|
||||
.expect("create multipart staging directory");
|
||||
tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
|
||||
.await
|
||||
.expect("replace one upload parent directory with a file");
|
||||
multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
|
||||
.await
|
||||
.expect_err("the affected disk must fail directory enumeration");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing still accepts the remaining discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
|
||||
.await
|
||||
.expect("remove the invalid directory entry");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after repair");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_data_movement_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-data-movement";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
|
||||
);
|
||||
set_disks
|
||||
.new_multipart_upload(
|
||||
bucket,
|
||||
"object.bin",
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create data movement upload");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_unreadable_ownership() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-corrupt";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let mut corrupted = 0;
|
||||
for temp_dir in &temp_dirs {
|
||||
for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
tokio::fs::write(path, b"invalid upload metadata")
|
||||
.await
|
||||
.expect("corrupt upload metadata");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "upload metadata must exist before corruption");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("unreadable upload ownership must block cleanup");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
|
||||
@@ -46,9 +46,9 @@ use super::super::{
|
||||
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
|
||||
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
|
||||
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer_with_orig_volume, debug,
|
||||
delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
|
||||
finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, delete_file_info_version_id,
|
||||
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
|
||||
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
|
||||
@@ -4459,10 +4459,9 @@ impl SetDisks {
|
||||
if let Some(disk) = disk_op
|
||||
&& disk.is_online().await
|
||||
{
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
match create_bitrot_writer(
|
||||
is_inline_buffer,
|
||||
Some(disk),
|
||||
bucket,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_obj,
|
||||
shard_file_size,
|
||||
@@ -6129,11 +6128,7 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
let all_local_disks_cleaned_tmp = all_commit_disks_local
|
||||
&& online_disks.len() == commit_set.set_drive_count
|
||||
&& online_disks.iter().all(Option::is_some)
|
||||
&& !issue3031_diag_enabled();
|
||||
if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
|
||||
if !tail_owns_tmp_cleanup {
|
||||
let cleanup_set = commit_set.clone();
|
||||
let cleanup_tmp_dir = commit_tmp_dir.clone();
|
||||
cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
|
||||
@@ -11069,9 +11064,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if let Some(err) = &gerr
|
||||
&& goi.name.is_empty()
|
||||
{
|
||||
if (is_err_object_not_found(err) || is_err_version_not_found(err))
|
||||
&& should_force_delete_marker_for_missing_version(&opts)
|
||||
{
|
||||
if should_force_delete_marker_for_missing_version(&opts) {
|
||||
version_found = false;
|
||||
} else {
|
||||
if is_err_object_not_found(err) {
|
||||
|
||||
@@ -44,16 +44,15 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
|
||||
WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
|
||||
WalkVersionsSortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -523,22 +523,7 @@ impl ECStore {
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
|
||||
}
|
||||
|
||||
/// Validate an external fence and the bucket incarnation in one metadata transaction.
|
||||
pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
|
||||
@@ -37,8 +37,8 @@ use crate::error::{
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::list::{
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
|
||||
StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -80,21 +80,6 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthoritativeListOperations for ECStore {
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
self.walk_authoritatively(rx, bucket, prefix, result, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the versioned listing options used by recursive object walks.
|
||||
///
|
||||
/// `list_path_result` normally derives these scan bounds before dispatching to
|
||||
@@ -1307,37 +1292,6 @@ impl ECStore {
|
||||
);
|
||||
|
||||
let rx = rx.child_token();
|
||||
if self.pools.len() == 1 && self.pools[0].disk_set.len() == 1 {
|
||||
rustfs_io_metrics::record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 1, 1);
|
||||
let res = self.pools[0].disk_set[0].list_path(rx, opts, sender).await;
|
||||
let mut outcome = ListPathRawOutcome::default();
|
||||
let err = match res {
|
||||
Ok(set_outcome) => {
|
||||
outcome.limit_reached = set_outcome.limit_reached;
|
||||
None
|
||||
}
|
||||
Err(err) => Some(err),
|
||||
};
|
||||
let errs = [err];
|
||||
if is_all_not_found(&errs) {
|
||||
return Ok(outcome);
|
||||
}
|
||||
if let Some(err) = errs.into_iter().flatten().next() {
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"store_list_objects_list_merged",
|
||||
merge_started.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
debug!(
|
||||
set_pair_count = 1,
|
||||
limit_reached = outcome.limit_reached,
|
||||
error_count = 0,
|
||||
"store list_merged finished"
|
||||
);
|
||||
return Ok(outcome);
|
||||
}
|
||||
|
||||
let mut futures = Vec::new();
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
@@ -1565,57 +1519,6 @@ impl ECStore {
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn walk_authoritatively(
|
||||
self: Arc<Self>,
|
||||
cancellation: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
let cancel = cancellation.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let mut scans = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
inputs.push(receiver);
|
||||
let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
|
||||
// Internal absence checks must include reserved object prefixes.
|
||||
scan.skip_hidden_prefix_check = true;
|
||||
scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
|
||||
}
|
||||
}
|
||||
if scans.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
|
||||
let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
|
||||
let scan_results = join_all(scans).await;
|
||||
let merge_result = merge.await.map_err(Error::from)?;
|
||||
let forward_result = forwarder.await.map_err(Error::from);
|
||||
|
||||
// An unavailable set cannot establish absence, even when another set
|
||||
// completed. Only an explicitly missing directory or volume is empty.
|
||||
for scan in scan_results {
|
||||
match scan {
|
||||
Ok(outcome) if !outcome.limit_reached => {}
|
||||
Ok(_) => return Err(StorageError::ErasureReadQuorum),
|
||||
Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
merge_result?;
|
||||
forward_result?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(Error::other("authoritative object walk was cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
@@ -1908,7 +1811,7 @@ async fn gather_results(
|
||||
if opts.versioned
|
||||
&& is_object
|
||||
&& entry
|
||||
.xl_meta_ref()
|
||||
.xl_meta()
|
||||
.is_ok_and(|meta| !meta.versions.is_empty() && meta.versions.iter().all(|version| version.header.free_version()))
|
||||
{
|
||||
continue;
|
||||
@@ -2586,8 +2489,7 @@ impl SetDisks {
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
/// version uses the read quorum of its own erasure layout. Null versions
|
||||
/// in the system bucket also retain GET's commit-proof selection. A
|
||||
/// version is resolved at the read quorum of its own erasure layout; a
|
||||
/// version that cannot be ruled out fails the scan instead of being
|
||||
/// omitted. Merged `ListObjectVersions` pages go through `list_path`.
|
||||
async fn list_versions_authoritatively(
|
||||
@@ -2926,7 +2828,6 @@ mod test {
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
|
||||
use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use bytes::Bytes;
|
||||
@@ -5787,126 +5688,6 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
for new_etag in ["new", "old"] {
|
||||
let new =
|
||||
test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
|
||||
let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
|
||||
|
||||
assert!(matches!(
|
||||
SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_selects_committed_null_version() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
|
||||
.collect(),
|
||||
);
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
|
||||
.expect("committed generation should resolve")
|
||||
.expect("committed record should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (old_copies, new_copies, expected) in [
|
||||
(2, 1, Some("old")),
|
||||
(1, 2, None),
|
||||
(0, 3, Some("new")),
|
||||
(0, 2, Some("new")),
|
||||
(0, 1, None),
|
||||
] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| {
|
||||
if slot < old_copies {
|
||||
Some(old.clone())
|
||||
} else if slot < old_copies + new_copies {
|
||||
Some(new.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let errors = (0..4)
|
||||
.map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
|
||||
.collect::<Vec<_>>();
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
|
||||
|
||||
if let Some(expected) = expected {
|
||||
let info = resolved
|
||||
.expect("degraded committed record should resolve")
|
||||
.expect("degraded committed record should be present")
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
} else {
|
||||
assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for old_copies in [1, 2] {
|
||||
let entries = MetaCacheEntries(vec![
|
||||
Some(new.clone()),
|
||||
Some(new.clone()),
|
||||
Some(old.clone()),
|
||||
(old_copies == 2).then(|| old.clone()),
|
||||
]);
|
||||
let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
|
||||
.expect("user bucket read quorum should resolve")
|
||||
.expect("user bucket record should be present");
|
||||
let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
|
||||
let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
|
||||
let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
|
||||
.expect("explicit version read quorum should resolve")
|
||||
.expect("explicit version should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
|
||||
let entry = test_object_meta_entry_with_erasure_versions(
|
||||
@@ -6248,49 +6029,6 @@ mod test {
|
||||
Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
|
||||
let (_dirs, store) = single_set_store(16).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for (slot, disk) in set.disks.read().await.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("fixture drive is online");
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{name}/{STORAGE_FORMAT_FILE}"),
|
||||
versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
|
||||
)
|
||||
.await
|
||||
.expect("persist reserved metadata versions");
|
||||
}
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let walk = store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let collect = async {
|
||||
let mut versions = Vec::new();
|
||||
while let Some(item) = receiver.recv().await {
|
||||
assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
|
||||
let item = item.item.expect("walk must return an object");
|
||||
versions.push((item.name, item.version_id));
|
||||
}
|
||||
versions
|
||||
};
|
||||
let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
|
||||
.await
|
||||
.expect("authoritative reserved-prefix walk must terminate");
|
||||
walk.expect("authoritative walk must succeed");
|
||||
assert_eq!(
|
||||
versions,
|
||||
[
|
||||
(name.to_string(), Some(Uuid::from_u128(1))),
|
||||
(name.to_string(), Some(Uuid::from_u128(2)))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// A versioned DeleteObject(versionId) can reach write quorum while three
|
||||
// drives are offline; they keep the deleted version after they return. A
|
||||
// different drive going offline later must not turn that sub-quorum
|
||||
@@ -6361,16 +6099,6 @@ mod test {
|
||||
.await
|
||||
.expect("set walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
let walk =
|
||||
store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let drain = async { while receiver.recv().await.is_some() {} };
|
||||
let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
|
||||
.await
|
||||
.expect("authoritative store walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let recursive_delete_scan = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)
|
||||
|
||||
@@ -19,28 +19,10 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_utils::path::encode_dir_object_ref;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
|
||||
type Error = Error;
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
let (_, lifecycle_guard) = self
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await?;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.ensure_no_multipart_uploads(bucket).await?;
|
||||
}
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
|
||||
std::sync::OnceLock::new();
|
||||
@@ -1186,9 +1168,11 @@ impl ECStore {
|
||||
let oi = if require_all_pools {
|
||||
lookup_opts.metadata_cache_safe = false;
|
||||
lookup_opts.suppress_read_repair = true;
|
||||
// Only completion uses this branch, after validating bucket and object.
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
|
||||
if let Err(err) = check_object_args(bucket, object) {
|
||||
return Some(err);
|
||||
}
|
||||
let encoded_object = encode_dir_object(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
|
||||
.await
|
||||
.and_then(|(info, _)| {
|
||||
lookup_opts.precondition_check(&info)?;
|
||||
|
||||
@@ -5637,11 +5637,13 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if self.single_pool()
|
||||
&& !creates_latest_marker
|
||||
&& !opts.data_movement
|
||||
&& opts.http_preconditions.is_none()
|
||||
&& !opts.replication_request
|
||||
&& !opts.versioned
|
||||
&& !opts.version_suspended
|
||||
&& !opts.delete_marker
|
||||
&& opts.version_id.is_none()
|
||||
&& !self.is_suspended(0).await
|
||||
&& !self.is_pool_rebalancing(0).await
|
||||
{
|
||||
@@ -5653,10 +5655,6 @@ impl ECStore {
|
||||
.await
|
||||
{
|
||||
Ok(mut res) => {
|
||||
#[cfg(test)]
|
||||
if creates_latest_marker {
|
||||
pause_versioned_delete_marker_after_commit(bucket, object).await;
|
||||
}
|
||||
res.name = decode_dir_object(res.name.as_str());
|
||||
return Ok(res);
|
||||
}
|
||||
|
||||
@@ -2070,53 +2070,6 @@ mod test {
|
||||
assert!(meta.versions.iter().all(|version| !version.header.free_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bytes_count_toward_listing_limit_matches_fileinfo_visibility() {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut obj = FileInfo::new("object", 2, 2);
|
||||
obj.version_id = Some(Uuid::new_v4());
|
||||
obj.mod_time = Some(OffsetDateTime::now_utc());
|
||||
meta.add_version(obj.clone()).expect("add object version");
|
||||
let obj_bytes = meta.marshal_msg().expect("marshal object");
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(&obj_bytes));
|
||||
|
||||
let dm = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: obj.mod_time.map(|t| t + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
meta.delete_version(&dm).expect("add delete marker");
|
||||
let dm_bytes = meta.marshal_msg().expect("marshal delete marker");
|
||||
assert!(!FileMeta::bytes_count_toward_listing_limit(&dm_bytes));
|
||||
|
||||
let (_, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
source.version_id = Some(Uuid::new_v4());
|
||||
let mut transitioned = FileMeta::new();
|
||||
transitioned.add_version(source.clone()).expect("seed transitioned version");
|
||||
let mut del_req = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: source.version_id,
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
del_req.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
transitioned
|
||||
.delete_version(&del_req)
|
||||
.expect("delete transitioned version leaving free version");
|
||||
assert!(!FileMeta::bytes_count_toward_listing_limit(
|
||||
&transitioned.marshal_msg().expect("marshal free-only")
|
||||
));
|
||||
|
||||
transitioned.add_version(obj).expect("add live object version");
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(
|
||||
&transitioned.marshal_msg().expect("marshal free + live object")
|
||||
));
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(b"corrupt-xl-meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_uses_canonical_equal_time_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
|
||||
@@ -324,63 +324,6 @@ impl FileMeta {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn buffers_match_strict(self_buf: &[u8], other_buf: &[u8]) -> bool {
|
||||
let Ok((s_meta, _)) = Self::parse_validated_xl_envelope(self_buf) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_meta, _)) = Self::parse_validated_xl_envelope(other_buf) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if s_meta.is_empty() || o_meta.is_empty() {
|
||||
return s_meta.is_empty() && o_meta.is_empty();
|
||||
}
|
||||
|
||||
let Ok((s_len, s_header_ver, _s_meta_ver, s_meta)) = Self::decode_validated_xl_headers(s_meta) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_len, o_header_ver, _o_meta_ver, o_meta)) = Self::decode_validated_xl_headers(o_meta) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if s_len != o_len {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut s_cur: Cursor<&[u8]> = Cursor::new(s_meta);
|
||||
let mut o_cur: Cursor<&[u8]> = Cursor::new(o_meta);
|
||||
for _ in 0..s_len {
|
||||
let Ok((s_header_raw, _)) = Self::read_next_raw_slices(s_meta, &mut s_cur) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_header_raw, _)) = Self::read_next_raw_slices(o_meta, &mut o_cur) else {
|
||||
return false;
|
||||
};
|
||||
if s_header_ver == XL_HEADER_VERSION
|
||||
&& o_header_ver == XL_HEADER_VERSION
|
||||
&& s_header_raw.len() == 38
|
||||
&& o_header_raw.len() == 38
|
||||
&& s_header_raw == o_header_raw
|
||||
&& parse_canonical_v3_header_fields(s_header_raw).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut s_header = FileMetaVersionHeader::default();
|
||||
if s_header.unmarshal_v(s_header_ver, s_header_raw).is_err() {
|
||||
return false;
|
||||
}
|
||||
let mut o_header = FileMetaVersionHeader::default();
|
||||
if o_header.unmarshal_v(o_header_ver, o_header_raw).is_err() {
|
||||
return false;
|
||||
}
|
||||
if !Self::headers_match_strict(&s_header, &o_header) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn header_matches_canonical_v3_raw_strict(s_header: &FileMetaVersionHeader, buf: &[u8]) -> Option<bool> {
|
||||
if buf.len() != 38 {
|
||||
@@ -682,49 +625,6 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bytes_count_toward_listing_limit(buf: &[u8]) -> bool {
|
||||
let Ok((meta, _inline_buf)) = Self::parse_validated_xl_envelope(buf) else {
|
||||
return true;
|
||||
};
|
||||
if meta.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Ok((versions_len, header_ver, _meta_ver, meta)) = Self::decode_validated_xl_headers(meta) else {
|
||||
return true;
|
||||
};
|
||||
if versions_len == 0 {
|
||||
return false;
|
||||
}
|
||||
let mut cur = Cursor::new(meta);
|
||||
let mut selected: Option<(FileMetaVersionHeader, &[u8])> = None;
|
||||
let mut any_non_hidden = false;
|
||||
for _ in 0..versions_len {
|
||||
let Ok((header, ver_meta)) = Self::read_next_shallow_slice(meta, &mut cur, header_ver) else {
|
||||
return true;
|
||||
};
|
||||
if header.version_type != VersionType::Delete && !header.free_version() {
|
||||
any_non_hidden = true;
|
||||
}
|
||||
if selected.is_some() {
|
||||
continue;
|
||||
}
|
||||
if header.free_version() && header.version_id.is_some_and(|id| !id.is_nil()) {
|
||||
continue;
|
||||
}
|
||||
selected = Some((header, ver_meta));
|
||||
}
|
||||
let Some((header, ver_meta)) = selected else {
|
||||
return false;
|
||||
};
|
||||
if !header.free_version() && header.version_type != VersionType::Delete {
|
||||
return true;
|
||||
}
|
||||
if !any_non_hidden {
|
||||
return false;
|
||||
}
|
||||
FileMetaVersion::decode_fileinfo_from_slice(ver_meta, "", "", true, false).is_err()
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "FileMeta")]
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
let mut wr =
|
||||
|
||||
@@ -352,22 +352,6 @@ impl MetaCacheEntry {
|
||||
(prefer, true)
|
||||
}
|
||||
|
||||
pub fn matches_strict_borrowed(&mut self, other: &MetaCacheEntry) -> bool {
|
||||
if self.name != other.name {
|
||||
return false;
|
||||
}
|
||||
|
||||
if other.is_dir() || self.is_dir() {
|
||||
return other.is_dir() == self.is_dir();
|
||||
}
|
||||
|
||||
if self.cached.is_none() && other.cached.is_none() {
|
||||
return FileMeta::buffers_match_strict(&self.metadata, &other.metadata);
|
||||
}
|
||||
|
||||
self.matches_strict(other)
|
||||
}
|
||||
|
||||
pub fn matches_strict(&mut self, other: &MetaCacheEntry) -> bool {
|
||||
if self.name != other.name {
|
||||
return false;
|
||||
@@ -1005,17 +989,6 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn write_obj_buffered(&mut self, obj: &MetaCacheEntry) -> Result<()> {
|
||||
self.init().await?;
|
||||
|
||||
Self::encode_obj(&mut self.buf, obj)?;
|
||||
if self.buf.len() >= METACACHE_READ_CHUNK_SIZE {
|
||||
self.flush().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn close(&mut self) -> Result<()> {
|
||||
self.close_with_limit_reached(false).await
|
||||
}
|
||||
@@ -1309,85 +1282,6 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
|
||||
}
|
||||
|
||||
pub fn skip_current(&mut self) -> bool {
|
||||
if self.has_current && self.init && self.err.is_none() {
|
||||
self.has_current = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_peek_buffered(&mut self) -> Option<Result<Option<MetaCacheEntry>>> {
|
||||
if !self.init || self.err.is_some() || self.terminated {
|
||||
return None;
|
||||
}
|
||||
|
||||
let avail = &self.buf[self.pos..];
|
||||
let &first = avail.first()?;
|
||||
if first != 0xc3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let str_marker = *avail.get(1)?;
|
||||
let (str_hdr, str_len) = match str_marker {
|
||||
0xa0..=0xbf => (1usize, usize::from(str_marker & 0x1f)),
|
||||
0xd9 => (2usize, usize::from(*avail.get(2)?)),
|
||||
0xda => (3usize, usize::from(u16::from_be_bytes([*avail.get(2)?, *avail.get(3)?]))),
|
||||
0xdb => (
|
||||
5usize,
|
||||
usize::try_from(u32::from_be_bytes([*avail.get(2)?, *avail.get(3)?, *avail.get(4)?, *avail.get(5)?])).ok()?,
|
||||
),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let bin_offset = 1usize.checked_add(str_hdr)?.checked_add(str_len)?;
|
||||
let bin_marker = *avail.get(bin_offset)?;
|
||||
let (bin_hdr, bin_len) = match bin_marker {
|
||||
0xc4 => (2usize, usize::from(*avail.get(bin_offset + 1)?)),
|
||||
0xc5 => (
|
||||
3usize,
|
||||
usize::from(u16::from_be_bytes([*avail.get(bin_offset + 1)?, *avail.get(bin_offset + 2)?])),
|
||||
),
|
||||
0xc6 => (
|
||||
5usize,
|
||||
usize::try_from(u32::from_be_bytes([
|
||||
*avail.get(bin_offset + 1)?,
|
||||
*avail.get(bin_offset + 2)?,
|
||||
*avail.get(bin_offset + 3)?,
|
||||
*avail.get(bin_offset + 4)?,
|
||||
]))
|
||||
.ok()?,
|
||||
),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let total_len = bin_offset.checked_add(bin_hdr)?.checked_add(bin_len)?;
|
||||
if avail.len() < total_len {
|
||||
return None;
|
||||
}
|
||||
|
||||
let Ok(decoded) = from_utf8(&avail[1 + str_hdr..bin_offset]) else {
|
||||
return None;
|
||||
};
|
||||
let name = decoded.to_owned();
|
||||
let metadata = if bin_len == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
avail[bin_offset + bin_hdr..total_len].to_vec()
|
||||
};
|
||||
|
||||
self.pos += total_len;
|
||||
self.reset();
|
||||
self.has_current = true;
|
||||
Some(Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
metadata,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn skip(&mut self, size: usize) -> Result<()> {
|
||||
self.check_init().await?;
|
||||
|
||||
@@ -4363,95 +4257,4 @@ mod tests {
|
||||
.expect("resolve mixed empty");
|
||||
assert_eq!(res_mixed_empty.cached.expect("cached").versions.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_strict_borrowed_matches_shallow_headers_and_rejects_corrupt_or_divergent_buffers() {
|
||||
let base_bytes = create_real_xlmeta().expect("create real xlmeta");
|
||||
let mut peer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
|
||||
// Mutate inner version payload (ver.meta) without changing shallow header.
|
||||
let mut ver = FileMetaVersion::try_from(peer_fm.versions[1].meta.as_slice()).expect("decode version");
|
||||
ver.object.as_mut().expect("object version").erasure_index += 1;
|
||||
peer_fm.versions[1].meta = ver.marshal_msg().expect("marshal peer version");
|
||||
let peer_bytes = peer_fm.marshal_msg().expect("marshal peer xlmeta");
|
||||
assert_ne!(base_bytes, peer_bytes, "peer shard has distinct erasure index payload");
|
||||
|
||||
let mut e1 = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: base_bytes.clone(),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let e1_clone = e1.clone();
|
||||
let e2 = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: peer_bytes,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
assert!(e1.matches_strict_borrowed(&e2));
|
||||
assert!(e1.matches_strict_borrowed(&e1_clone));
|
||||
|
||||
// Different shallow header must not match.
|
||||
let mut newer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
|
||||
newer_fm.versions[0].header.version_id = Some(Uuid::new_v4());
|
||||
let newer_entry = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: newer_fm.marshal_msg().expect("marshal newer xlmeta"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
assert!(!e1.matches_strict_borrowed(&newer_entry));
|
||||
|
||||
// Identical corrupt buffers must return false, never true via byte-equality shortcut.
|
||||
let mut corrupt = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: b"XL2 corrupt-payload".to_vec(),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let corrupt_peer = corrupt.clone();
|
||||
assert!(!corrupt.matches_strict_borrowed(&corrupt_peer));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_peek_buffered_and_skip_current_match_async_peek_and_skip() {
|
||||
let entries: Vec<_> = (0..5)
|
||||
.map(|i| MetaCacheEntry {
|
||||
name: format!("buffered-key-{i}"),
|
||||
metadata: vec![i as u8; 16],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})
|
||||
.collect();
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut buf);
|
||||
writer.write(&entries).await.expect("write entries");
|
||||
writer.close_with_limit_reached(false).await.expect("close stream");
|
||||
|
||||
let mut reader = MetacacheReader::new(Cursor::new(buf.into_inner()));
|
||||
assert!(reader.try_peek_buffered().is_none(), "uninitialized reader must defer to async peek");
|
||||
assert!(!reader.skip_current(), "no current entry before first peek");
|
||||
|
||||
let first = reader.peek().await.expect("first peek").expect("entry 0");
|
||||
assert_eq!(first, entries[0]);
|
||||
assert!(reader.skip_current(), "skip_current must consume the peeked entry");
|
||||
assert!(!reader.skip_current(), "second skip_current without peek must return false");
|
||||
|
||||
for expected in &entries[1..] {
|
||||
let next = reader
|
||||
.try_peek_buffered()
|
||||
.expect("remaining entries are already buffered")
|
||||
.expect("valid entry")
|
||||
.expect("present entry");
|
||||
assert_eq!(&next, expected);
|
||||
assert!(reader.skip_current());
|
||||
}
|
||||
|
||||
assert!(
|
||||
reader.try_peek_buffered().is_none(),
|
||||
"stream terminator must defer to async peek for trailer handling"
|
||||
);
|
||||
assert!(reader.peek().await.expect("eof peek").is_none());
|
||||
assert_eq!(reader.limit_reached(), Some(false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,9 +90,7 @@ pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::MultipartUploadInspection;
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::AuthoritativeListOperations;
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
pub use object::ObjectToDeleteIdentity;
|
||||
|
||||
@@ -15,15 +15,6 @@
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Checks upload absence before removing a bucket's data protections.
|
||||
#[async_trait::async_trait]
|
||||
pub trait MultipartUploadInspection: Send + Sync {
|
||||
type Error: std::error::Error + Send + Sync + 'static;
|
||||
|
||||
/// Unknown or unreadable upload ownership must fail this check.
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct MultipartUploadResult {
|
||||
pub upload_id: String,
|
||||
|
||||
@@ -566,29 +566,6 @@ pub trait ListOperations: Send + Sync + fmt::Debug {
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
/// Quorum-backed enumeration for decisions that require proving object absence.
|
||||
///
|
||||
/// Consumers must drain the result stream and await successful completion before
|
||||
/// trusting the scan. An error or cancellation can follow partial results and
|
||||
/// leaves absence unproven. Callers coordinate concurrent mutations separately;
|
||||
/// this capability does not provide a snapshot or retain a mutation lock.
|
||||
#[async_trait::async_trait]
|
||||
pub trait AuthoritativeListOperations: ListOperations {
|
||||
/// Resolves every set at its version read quorum, including reserved prefixes.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when a set cannot establish quorum, metadata cannot be
|
||||
/// resolved, or enumeration is interrupted. Retry starts a new scan.
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: Self::WalkCancellation,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub trait MultipartOperations: Send + Sync + fmt::Debug {
|
||||
|
||||
@@ -91,26 +91,6 @@ Use `GET /iceberg/v1/buckets/{warehouse}` to check `enabled` and `disable-suppor
|
||||
|
||||
Review the bucket lifecycle configuration before disabling: clearing table-bucket protection allows existing object expiration rules to run again. Objects are preserved by the disable operation itself. Catalog disablement requires no namespaces, retained table/view resources, or objects under `.rustfs-table/`; an active rename or backing migration also blocks it. Clean up through supported catalog operations; do not manually delete internal metadata to bypass these checks.
|
||||
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. A plain DELETE and a subsequent PUT preserve table metadata and object data.
|
||||
|
||||
### Clean Up Deleted Empty Tables
|
||||
|
||||
For the object-backed catalog, `DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes the metadata and internal records left by deleted empty tables, then disables the table catalog. Ordinary table deletion continues to preserve files. Cleanup preserves existing table record formats and can process empty tables deleted before the server upgrade.
|
||||
|
||||
Complete the server upgrade across all serving nodes and set the existing `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` setting before requesting cleanup. Delete the empty tables and their namespaces through the catalog APIs, then send the cleanup request with `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. The `/_iceberg/v1` alias accepts the same option. Omitting `cleanup` or setting it to `false` keeps the ordinary disable behavior; repeated or invalid values are rejected.
|
||||
|
||||
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active catalog resources, snapshots or other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete cleanup plan and object permissions before deleting any object. Durable-strong catalogs retain their existing disable procedure.
|
||||
|
||||
If a storage operation fails during cleanup, retry the same request. The server checks the remaining objects again and keeps the bucket's table protection enabled until cleanup and catalog disablement complete. No separate cleanup task file is created. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
|
||||
|
||||
## Object-Backed Registration Reads
|
||||
|
||||
Table registration checks table ID uniqueness and warehouse directory overlap using the existing catalog and warehouse index records. Each registration obtains a complete inventory under the existing publication and migration guards. System-bucket inventory records must satisfy the same metadata commit checks as ordinary catalog reads. An incomplete or failed inventory rejects registration.
|
||||
|
||||
Each server keeps a bounded memory cache of validated ownership records. A record can be reused only when its object identity and ETag match the current inventory. New or changed records are read and validated again. Cache eviction and process restarts cause additional reads; catalog state continues to come from the existing persisted records. The cache is automatic and requires no additional configuration or stored objects.
|
||||
|
||||
Registration still enumerates the namespace and warehouse index prefixes, so enumeration work grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A cache hit verifies the current metadata identity and reuses previously validated content; it does not re-read the current body to check its physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication.
|
||||
|
||||
Existing catalog formats and Console and Iceberg client request formats are preserved. Concurrent writers must follow the existing publication, migration, and storage fencing protocol; the memory cache does not provide synchronization for writers that bypass those guards.
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. No table metadata or object data is deleted by either state change.
|
||||
|
||||
Before rolling back to a server version without this endpoint, re-enable disabled catalogs with PUT while the supporting version is running. Older enable handlers do not reactivate inactive catalog entries. Complete the server upgrade across the cluster before using disablement.
|
||||
|
||||
Generated
+6
-6
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1793466162,
|
||||
"narHash": "sha256-pt+peAEoOK3lLLdtbnQ1AGIBTZbdB6jrkgmurfcQwig=",
|
||||
"lastModified": 1792812153,
|
||||
"narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a0788123ada57ec92a79ab2581cda6d3c6cca75f",
|
||||
"rev": "674a2028661e550a97400c1cc992b9cca9a668a1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -29,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1793434469,
|
||||
"narHash": "sha256-+sgVNI2c67di+XIeMaz5KSei4T65kX68s5Nl0E5Gzbc=",
|
||||
"lastModified": 1792825530,
|
||||
"narHash": "sha256-aphXA2IFNd3q48++TKmfuuvDDjkrgGdJDnDIhij9684=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "6ecf534a7c16e7c969f26838fdd8c93597ae93f3",
|
||||
"rev": "5b9f1198b59bcb31c5c3250b332e7d77590e6306",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -43,9 +43,9 @@ mod tests {
|
||||
.map(|value| value.to_str().expect("allow-origin header must be valid UTF-8").to_string())
|
||||
}
|
||||
|
||||
/// Sends a brotli-encoded response through `buffer_brotli_responses` and returns
|
||||
/// the `Content-Length` header value and body bytes.
|
||||
async fn run_brotli_middleware(body: &'static str, has_content_length: bool) -> (Option<String>, bytes::Bytes) {
|
||||
/// Runs a request through a router that returns a brotli-encoded response without
|
||||
/// `Content-Length` and returns the response's `content-length` header value, if any.
|
||||
async fn brotli_response_content_length(has_content_length: bool) -> Option<String> {
|
||||
use crate::admin::console::buffer_brotli_responses;
|
||||
use axum::middleware;
|
||||
use axum::{Router, body::Body, routing::get};
|
||||
@@ -57,9 +57,9 @@ mod tests {
|
||||
get(move || async move {
|
||||
let mut builder = http::Response::builder().header(http::header::CONTENT_ENCODING, "br");
|
||||
if has_content_length {
|
||||
builder = builder.header(http::header::CONTENT_LENGTH, body.len().to_string());
|
||||
builder = builder.header(http::header::CONTENT_LENGTH, "4");
|
||||
}
|
||||
builder.body(Body::from(body)).expect("response must build")
|
||||
builder.body(Body::from("data")).expect("response must build")
|
||||
}),
|
||||
)
|
||||
.layer(middleware::from_fn(buffer_brotli_responses));
|
||||
@@ -74,130 +74,27 @@ mod tests {
|
||||
.await
|
||||
.expect("request must not fail");
|
||||
|
||||
let content_length = response
|
||||
response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.map(|value| value.to_str().expect("content-length header must be valid UTF-8").to_string());
|
||||
let body_bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("body must be readable");
|
||||
(content_length, body_bytes)
|
||||
.map(|value| value.to_str().expect("content-length header must be valid UTF-8").to_string())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_buffer_brotli_responses() {
|
||||
// Brotli response without Content-Length must have Content-Length added with correct value.
|
||||
let (content_length, body) = run_brotli_middleware("fake-brotli-data", false).await;
|
||||
assert_eq!(
|
||||
content_length.as_deref(),
|
||||
Some("16"),
|
||||
"brotli response without Content-Length must have Content-Length set to the body length"
|
||||
);
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
b"fake-brotli-data",
|
||||
"body bytes must be preserved through the buffer round-trip"
|
||||
// Brotli response without Content-Length must have Content-Length added.
|
||||
assert!(
|
||||
brotli_response_content_length(false).await.is_some(),
|
||||
"brotli response without Content-Length must have Content-Length added"
|
||||
);
|
||||
|
||||
// Brotli response that already carries Content-Length must pass through unchanged.
|
||||
let (content_length, body) = run_brotli_middleware("data", true).await;
|
||||
assert_eq!(
|
||||
content_length.as_deref(),
|
||||
brotli_response_content_length(true).await.as_deref(),
|
||||
Some("4"),
|
||||
"brotli response that already has Content-Length must not be re-buffered"
|
||||
);
|
||||
assert_eq!(body.as_ref(), b"data", "body bytes must be preserved when passing through unchanged");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_buffer_brotli_responses_non_brotli_passthrough() {
|
||||
use crate::admin::console::buffer_brotli_responses;
|
||||
use axum::middleware;
|
||||
use axum::{Router, body::Body, routing::get};
|
||||
use tower::ServiceExt;
|
||||
|
||||
// Non-brotli response must pass through with no modification.
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/",
|
||||
get(|| async {
|
||||
http::Response::builder()
|
||||
.header(http::header::CONTENT_ENCODING, "gzip")
|
||||
.body(Body::from("gzip-data"))
|
||||
.expect("response must build")
|
||||
}),
|
||||
)
|
||||
.layer(middleware::from_fn(buffer_brotli_responses));
|
||||
|
||||
let response = app
|
||||
.oneshot(
|
||||
http::Request::builder()
|
||||
.uri("/")
|
||||
.body(Body::empty())
|
||||
.expect("request must build"),
|
||||
)
|
||||
.await
|
||||
.expect("request must not fail");
|
||||
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_ENCODING)
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
Some("gzip"),
|
||||
"non-brotli Content-Encoding must pass through unchanged"
|
||||
);
|
||||
let body_bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("body must be readable");
|
||||
assert_eq!(body_bytes.as_ref(), b"gzip-data", "body bytes must pass through unchanged");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_buffer_brotli_responses_body_stream_error() {
|
||||
use crate::admin::console::buffer_brotli_responses;
|
||||
use axum::middleware;
|
||||
use axum::{Router, body::Body, routing::get};
|
||||
use tower::ServiceExt;
|
||||
|
||||
// A brotli response whose body stream errors must have Content-Encoding stripped
|
||||
// so the client does not attempt to decode an empty body as brotli.
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/",
|
||||
get(|| async {
|
||||
let broken = futures::stream::once(async {
|
||||
Err::<bytes::Bytes, std::io::Error>(std::io::Error::other("simulated stream error"))
|
||||
});
|
||||
http::Response::builder()
|
||||
.header(http::header::CONTENT_ENCODING, "br")
|
||||
.body(Body::from_stream(broken))
|
||||
.expect("response must build")
|
||||
}),
|
||||
)
|
||||
.layer(middleware::from_fn(buffer_brotli_responses));
|
||||
|
||||
let response = app
|
||||
.oneshot(
|
||||
http::Request::builder()
|
||||
.uri("/")
|
||||
.body(Body::empty())
|
||||
.expect("request must build"),
|
||||
)
|
||||
.await
|
||||
.expect("request must not fail");
|
||||
|
||||
assert!(
|
||||
response.headers().get(http::header::CONTENT_ENCODING).is_none(),
|
||||
"Content-Encoding must be stripped when body buffering fails so the client does not decode an empty body as brotli"
|
||||
);
|
||||
let body_bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("body must be readable");
|
||||
assert!(body_bytes.is_empty(), "body must be empty after buffering failure");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -50,23 +50,17 @@ pub struct DisableTableBucketHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for DisableTableBucketHandler {
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?;
|
||||
let cleanup = table_bucket_cleanup_requested_from_query(&req.uri)?;
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?;
|
||||
let app_context = runtime_sources::app_context_from_extensions(&req.extensions);
|
||||
let backend = table_catalog_backend_from_extensions(&req.extensions)?;
|
||||
let object_store = runtime_sources::object_store_from_req(&req)
|
||||
.ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?;
|
||||
let store = table_catalog_store_from_backend(backend.clone())?;
|
||||
install_table_catalog_s3_request_info(&mut req, &principal)?;
|
||||
let publication = TableCommitObjectBackend::for_request(backend, req);
|
||||
let response = if cleanup {
|
||||
cleanup_empty_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?
|
||||
} else {
|
||||
disable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?
|
||||
};
|
||||
let publication = TableCommitObjectBackend::preauthorized(backend);
|
||||
let response = disable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?;
|
||||
crate::app::bucket_usecase::notify_table_bucket_metadata_reload(warehouse.clone(), "disable table bucket", app_context)
|
||||
.await;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
|
||||
@@ -2092,25 +2092,17 @@ fn warehouse_from_config_query(uri: &http::Uri) -> S3Result<Option<String>> {
|
||||
}
|
||||
|
||||
fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
table_catalog_boolean_query(uri, "purgeRequested")
|
||||
}
|
||||
|
||||
fn table_bucket_cleanup_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
table_catalog_boolean_query(uri, "cleanup")
|
||||
}
|
||||
|
||||
fn table_catalog_boolean_query(uri: &http::Uri, parameter: &str) -> S3Result<bool> {
|
||||
let mut requested = None;
|
||||
let mut purge_requested = None;
|
||||
if let Some(query) = uri.query() {
|
||||
for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||
if key != parameter {
|
||||
if key != "purgeRequested" {
|
||||
continue;
|
||||
}
|
||||
if requested.is_some() {
|
||||
if purge_requested.is_some() {
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_BAD_REQUEST,
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("{parameter} query parameter must not be repeated"),
|
||||
"purgeRequested query parameter must not be repeated",
|
||||
));
|
||||
}
|
||||
let value = if value.eq_ignore_ascii_case("true") {
|
||||
@@ -2121,13 +2113,13 @@ fn table_catalog_boolean_query(uri: &http::Uri, parameter: &str) -> S3Result<boo
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_BAD_REQUEST,
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("{parameter} query parameter must be true or false"),
|
||||
"purgeRequested query parameter must be true or false",
|
||||
));
|
||||
};
|
||||
requested = Some(value);
|
||||
purge_requested = Some(value);
|
||||
}
|
||||
}
|
||||
Ok(requested.unwrap_or(false))
|
||||
Ok(purge_requested.unwrap_or(false))
|
||||
}
|
||||
|
||||
fn rest_table_snapshot_selection_from_query(uri: &http::Uri) -> S3Result<RestTableSnapshotSelection> {
|
||||
@@ -2425,10 +2417,9 @@ fn table_catalog_backend_from_extensions(
|
||||
) -> S3Result<crate::table_catalog::EcStoreTableCatalogObjectBackend<ECStore>> {
|
||||
let context = runtime_sources::app_context_from_extensions(extensions)
|
||||
.ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?;
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_runtimes(
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
context.object_store(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
context.table_catalog_registration_cache(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -2581,61 +2572,6 @@ where
|
||||
table_bucket_response(store, bucket, true).await
|
||||
}
|
||||
|
||||
async fn cleanup_empty_table_bucket_response<B>(
|
||||
store: &crate::table_catalog::ConfiguredTableCatalogStore<B>,
|
||||
publication: &TableCommitObjectBackend<B>,
|
||||
object_store: &ECStore,
|
||||
bucket: &str,
|
||||
) -> S3Result<TableBucketResponse>
|
||||
where
|
||||
B: crate::table_catalog::TableCatalogObjectBackend,
|
||||
{
|
||||
if !publication.publication_fence_fleet_confirmed {
|
||||
return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Conflict(
|
||||
"empty table bucket cleanup requires publication fences on every cluster node".to_string(),
|
||||
)));
|
||||
}
|
||||
crate::table_catalog::TableCommitPublication::begin_table_bucket(publication, bucket)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
|
||||
let incarnation = object_store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.map_err(|err| ApiError::other(format!("failed to read table bucket identity: {err}")))?;
|
||||
let prepared = store
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(bucket), publication)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
let mut cleanup = publication.finish(prepared).await?;
|
||||
for object in cleanup.metadata_objects() {
|
||||
if let Err(err) = publication.authorize(bucket, object, S3Action::DeleteObjectAction).await {
|
||||
return publication.finish(Err(catalog_store_error(err))).await;
|
||||
}
|
||||
}
|
||||
store
|
||||
.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
let fence = cleanup.mutation_fence();
|
||||
object_store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG,
|
||||
Vec::new(),
|
||||
incarnation,
|
||||
|| {
|
||||
fence
|
||||
.ensure_held()
|
||||
.map_err(|err| crate::admin::storage_api::error::StorageError::other(err.to_string()))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ApiError::other(format!("failed to disable table bucket: {err}")))?;
|
||||
drop(cleanup);
|
||||
table_bucket_response(store, bucket, false).await
|
||||
}
|
||||
|
||||
async fn disable_table_bucket_response<S, B>(
|
||||
store: &S,
|
||||
publication: &TableCommitObjectBackend<B>,
|
||||
|
||||
@@ -87,15 +87,6 @@ async fn call_load_table_handler(request: S3Request<Body>) -> S3Result<S3Respons
|
||||
RestLoadTableHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_disable_table_bucket_handler(request: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router.insert("/iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
router.insert("/_iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
let path = request.uri.path().to_string();
|
||||
let matched = router.at(&path).unwrap();
|
||||
DisableTableBucketHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_maintenance_worker_handler(request: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router
|
||||
@@ -1264,28 +1255,6 @@ fn drop_table_purge_query_is_explicit_and_strict() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_bucket_cleanup_query_is_explicit_and_strict() {
|
||||
for (uri, expected) in [
|
||||
("/iceberg/v1/buckets/analytics", false),
|
||||
("/iceberg/v1/buckets/analytics?cleanup=false", false),
|
||||
("/iceberg/v1/buckets/analytics?cleanup=true", true),
|
||||
("/_iceberg/v1/buckets/analytics?cleanup=true", true),
|
||||
] {
|
||||
assert_eq!(table_bucket_cleanup_requested_from_query(&uri.parse().unwrap()).unwrap(), expected);
|
||||
}
|
||||
for query in [
|
||||
"cleanup=",
|
||||
"cleanup=1",
|
||||
"cleanup=yes",
|
||||
"cleanup=true&cleanup=false",
|
||||
"cleanup=true&cleanup=true",
|
||||
] {
|
||||
let uri = format!("/iceberg/v1/buckets/analytics?{query}").parse().unwrap();
|
||||
assert!(table_bucket_cleanup_requested_from_query(&uri).is_err(), "invalid query: {query}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_conflicts_use_operation_specific_iceberg_errors() {
|
||||
let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict(
|
||||
@@ -1720,7 +1689,9 @@ fn table_bucket_toggle_handlers_reload_peer_metadata_after_publication() {
|
||||
(enable, "enable_table_bucket_response", "enable table bucket"),
|
||||
(disable, "disable_table_bucket_response", "disable table bucket"),
|
||||
] {
|
||||
let response_offset = block.find(response).expect("table bucket toggle should publish local state");
|
||||
let response_offset = block
|
||||
.find(&format!("let response = {response}"))
|
||||
.expect("table bucket toggle should publish local state");
|
||||
let reload = block
|
||||
.find(&format!("notify_table_bucket_metadata_reload(warehouse.clone(), \"{operation}\""))
|
||||
.expect("table bucket toggle should reload peer metadata");
|
||||
@@ -16303,799 +16274,3 @@ async fn disable_table_bucket_preserves_objects_and_can_be_reenabled() {
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
}
|
||||
|
||||
async fn create_dropped_empty_tables_for_cleanup<S, B>(
|
||||
catalog: &S,
|
||||
backend: &B,
|
||||
object_store: &ECStore,
|
||||
bucket: &str,
|
||||
table_count: usize,
|
||||
) -> Vec<String>
|
||||
where
|
||||
S: TableCatalogStore + ?Sized,
|
||||
B: TableCatalogObjectBackend,
|
||||
{
|
||||
object_store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
enable_table_bucket_response(catalog, &TableCommitObjectBackend::trusted(backend.clone()), object_store, bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").unwrap();
|
||||
create_namespace_response(
|
||||
catalog,
|
||||
bucket,
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut metadata_objects = Vec::new();
|
||||
for number in 0..table_count {
|
||||
let name = format!("events_{number:02}");
|
||||
let request = serde_json::from_value(serde_json::json!({
|
||||
"name": name,
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
create_table_response(
|
||||
catalog,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
bucket,
|
||||
&namespace,
|
||||
request,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_objects.push(
|
||||
catalog
|
||||
.load_table(bucket, &namespace.public_name(), &name)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.metadata_location,
|
||||
);
|
||||
}
|
||||
for number in 0..table_count {
|
||||
catalog
|
||||
.drop_table(bucket, &namespace.public_name(), &format!("events_{number:02}"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
catalog.drop_namespace(bucket, &namespace.public_name()).await.unwrap();
|
||||
metadata_objects
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn object_catalog_registration_cache_checks_real_records_without_additional_objects() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{
|
||||
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, StrongTableCatalogRuntime, TableCatalogBackingMode,
|
||||
TableCatalogObjectPaths, TableCatalogStoreError,
|
||||
};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("registration-cache-{}", Uuid::new_v4().simple());
|
||||
let backend =
|
||||
EcStoreTableCatalogObjectBackend::new_with_strong_runtime(object_store.clone(), StrongTableCatalogRuntime::default());
|
||||
let catalog = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
object_store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
enable_table_bucket_response(&catalog, &TableCommitObjectBackend::trusted(backend.clone()), &object_store, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").unwrap();
|
||||
create_namespace_response(
|
||||
&catalog,
|
||||
&bucket,
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
let bucket_entry = paths.table_bucket_entry_path(&bucket);
|
||||
let catalog_prefix = format!("{}/", bucket_entry.rsplit_once('/').unwrap().0);
|
||||
let mut create_micros = Vec::new();
|
||||
for number in 0..100 {
|
||||
let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
let request = serde_json::from_value(serde_json::json!({
|
||||
"name": format!("events_{number:03}"),
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
let started = std::time::Instant::now();
|
||||
create_table_response(
|
||||
&store,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
&bucket,
|
||||
&namespace,
|
||||
request,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("fresh store instances share the runtime cache and create independent tables");
|
||||
create_micros.push(started.elapsed().as_micros());
|
||||
if matches!(number + 1, 20 | 100) {
|
||||
let before = backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap();
|
||||
let cold_backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let probe = ConfiguredTableCatalogStore::new_for_test(cold_backend, TableCatalogBackingMode::ObjectBacked);
|
||||
let mut duplicate = catalog
|
||||
.load_table(&bucket, "analytics", &format!("events_{number:03}"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
duplicate.table = "duplicate_probe".to_string();
|
||||
for cache_state in ["cold", "warm"] {
|
||||
let started = std::time::Instant::now();
|
||||
assert!(matches!(
|
||||
probe.create_table(duplicate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
eprintln!(
|
||||
"registration_check tables={} cache={} elapsed_us={}",
|
||||
number + 1,
|
||||
cache_state,
|
||||
started.elapsed().as_micros()
|
||||
);
|
||||
}
|
||||
assert_eq!(before, backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap());
|
||||
let mut recent = create_micros[create_micros.len() - 5..].to_vec();
|
||||
recent.sort_unstable();
|
||||
eprintln!("create_table_response tables={} last_five_median_us={}", number + 1, recent[2]);
|
||||
}
|
||||
}
|
||||
assert_eq!(catalog.list_all_tables(&bucket).await.unwrap().len(), 100);
|
||||
let namespace_entry = paths.namespace_entry_path(&bucket, &namespace);
|
||||
let index_prefix = paths.warehouse_index_entries_prefix(&bucket);
|
||||
let objects = backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap();
|
||||
assert!(objects.iter().all(|key| {
|
||||
key == &bucket_entry || key == &namespace_entry || key.starts_with(&index_prefix) || key.ends_with("/table-entry.json")
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_reclaims_twenty_dropped_tables_and_allows_bucket_deletion() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-{}", Uuid::new_v4().simple());
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 20).await;
|
||||
assert!(
|
||||
disable_table_bucket_response(
|
||||
&catalog,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
object_store.as_ref(),
|
||||
&bucket,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let publication = TableCommitObjectBackend::test(backend.clone(), authorizations.clone(), None);
|
||||
let result = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket).await;
|
||||
assert!(!publication.finish(result).await.unwrap().enabled);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
let authorized = authorizations.lock().await;
|
||||
assert!(authorized.contains(&(object.clone(), S3Action::GetObjectAction)));
|
||||
assert!(authorized.contains(&(object.clone(), S3Action::DeleteObjectAction)));
|
||||
}
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_validates_authorization_and_metadata_before_deleting() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-check-{}", Uuid::new_v4().simple());
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let publication = TableCommitObjectBackend::rolling_upgrade(backend.clone());
|
||||
assert!(
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let denied_object = metadata_objects.last().unwrap().clone();
|
||||
let publication = TableCommitObjectBackend::test(backend.clone(), authorizations, Some(denied_object.clone()));
|
||||
let result = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket).await;
|
||||
assert_eq!(publication.finish(result).await.unwrap_err().code(), &S3ErrorCode::AccessDenied);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let original = backend.read_object(&bucket, &denied_object).await.unwrap().unwrap().data;
|
||||
backend
|
||||
.put_object(
|
||||
&bucket,
|
||||
&denied_object,
|
||||
b"{".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
backend
|
||||
.put_object(&bucket, &denied_object, original, crate::table_catalog::TableCatalogPutPrecondition::Any)
|
||||
.await
|
||||
.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_handler_enforces_request_object_permissions() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(
|
||||
crate::table_catalog::ENV_TABLE_CATALOG_BACKING,
|
||||
Some(crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT),
|
||||
),
|
||||
(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-request-{}", Uuid::new_v4().simple());
|
||||
rustfs_iam::store::object::ObjectStore::new(object_store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let iam = rustfs_iam::build_iam_sys(object_store.clone()).await.unwrap();
|
||||
let access_key = "cleanup-request-user";
|
||||
let secret_key = "cleanup-request-user-secret";
|
||||
iam.create_user(
|
||||
access_key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: secret_key.to_string(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let context = Arc::new(AppContext::new(
|
||||
object_store.clone(),
|
||||
Arc::new(RequestIam { handle: iam.clone() }),
|
||||
Arc::new(RequestKms),
|
||||
));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: "cleanup-request-root".to_string(),
|
||||
secret_key: "cleanup-request-root-secret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let request = |prefix: &str| {
|
||||
let mut request = table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
request.method = Method::DELETE;
|
||||
request.uri = format!("{prefix}/v1/buckets/{bucket}?cleanup=true").parse().unwrap();
|
||||
request
|
||||
};
|
||||
let object_resource = format!("arn:aws:s3:::{bucket}/*");
|
||||
let first_metadata_resource = format!("arn:aws:s3:::{bucket}/{}", metadata_objects[0]);
|
||||
for (policy_name, object_statements, allowed) in [
|
||||
("cleanup-request-no-read", Vec::new(), false),
|
||||
(
|
||||
"cleanup-request-partial-delete",
|
||||
vec![
|
||||
serde_json::json!({"Effect":"Allow","Action":["s3:GetObject"],"Resource":[object_resource]}),
|
||||
serde_json::json!({"Effect":"Allow","Action":["s3:DeleteObject"],"Resource":[first_metadata_resource]}),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(
|
||||
"cleanup-request-full",
|
||||
vec![serde_json::json!({
|
||||
"Effect":"Allow",
|
||||
"Action":["s3:GetObject","s3:DeleteObject"],
|
||||
"Resource":[object_resource],
|
||||
})],
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let mut statements = vec![serde_json::json!({"Effect":"Allow","Action":["admin:SetTableBucket"]})];
|
||||
statements.extend(object_statements);
|
||||
let policy = serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap();
|
||||
iam.set_policy(policy_name, Policy::parse_config(&policy).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, policy_name)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = call_disable_table_bucket_handler(request(if allowed { "/_iceberg" } else { "/iceberg" })).await;
|
||||
if allowed {
|
||||
let response = result.expect("request-scoped object permissions should authorize cleanup");
|
||||
assert_eq!(response.output.0, StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(&response.output.1.bytes().unwrap()).unwrap();
|
||||
assert_eq!(body["enabled"], false);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
} else {
|
||||
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::AccessDenied);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
crate::table_catalog::TableCatalogEntryState::Active,
|
||||
);
|
||||
}
|
||||
}
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_after_marker_write_rejection() {
|
||||
use crate::table_catalog::TableCommitPublication;
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-marker-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::default();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
TableCommitPublication::begin_table_bucket(&publication, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let mut cleanup = catalog
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(&bucket), &publication)
|
||||
.await
|
||||
.unwrap();
|
||||
catalog
|
||||
.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation)
|
||||
.await
|
||||
.unwrap();
|
||||
let marker_result = object_store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
&bucket,
|
||||
crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG,
|
||||
Vec::new(),
|
||||
incarnation,
|
||||
|| {
|
||||
Err(crate::admin::storage_api::error::StorageError::other(
|
||||
"injected marker publication rejection",
|
||||
))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(marker_result.is_err());
|
||||
drop(cleanup);
|
||||
TableCommitPublication::complete(&publication);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
crate::table_catalog::TableCatalogEntryState::Deleted,
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_reads_incarnation_after_publication_wait() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-recreated-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::default();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 0).await;
|
||||
let original_incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path();
|
||||
let guard = backend.acquire_write_lock(&bucket, &publication_lock).await.unwrap();
|
||||
let attempts = backend.write_lock_acquisition_count(&bucket, &publication_lock).await;
|
||||
let cleanup_bucket = bucket.clone();
|
||||
let cleanup_store = object_store.clone();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let cleanup = tokio::spawn(async move {
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, cleanup_store.as_ref(), &cleanup_bucket).await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(5), async {
|
||||
while backend.write_lock_acquisition_count(&bucket, &publication_lock).await == attempts {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup should wait for its publication lock");
|
||||
|
||||
// Replace the real bucket while the in-memory publication backend holds cleanup at its lock boundary.
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
object_store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
enable_table_bucket_marker(object_store.as_ref(), &bucket).await.unwrap();
|
||||
let replacement_incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
assert_ne!(replacement_incarnation, original_incarnation);
|
||||
drop(guard);
|
||||
let response = tokio::time::timeout(StdDuration::from_secs(10), cleanup)
|
||||
.await
|
||||
.expect("cleanup should resume after publication becomes available")
|
||||
.unwrap()
|
||||
.expect("cleanup must use the incarnation observed while holding its publication lock");
|
||||
assert!(!response.enabled);
|
||||
assert_eq!(
|
||||
object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap(),
|
||||
replacement_incarnation
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_metadata_delete_failures_without_removing_protection() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogObjectPaths};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
for fail_after_delete in [false, true] {
|
||||
let bucket = format!("cleanup-retry-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let mut metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 3).await;
|
||||
metadata_objects.sort();
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
let index_prefix = paths.warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
assert!(indexes.len() >= metadata_objects.len());
|
||||
if fail_after_delete {
|
||||
backend.fail_after_next_delete(&bucket, &metadata_objects[1]).await;
|
||||
} else {
|
||||
backend.fail_delete_attempt(&bucket, &metadata_objects[1], 1).await;
|
||||
}
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let error = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("injected"));
|
||||
assert!(!backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert_eq!(backend.object_exists(&bucket, &metadata_objects[1]).await.unwrap(), !fail_after_delete);
|
||||
assert!(backend.object_exists(&bucket, &metadata_objects[2]).await.unwrap());
|
||||
assert_eq!(backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(), indexes);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let restarted = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&restarted, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(backend.list_objects(&bucket, "").await.unwrap().is_empty());
|
||||
assert!(
|
||||
backend
|
||||
.list_objects(RUSTFS_META_BUCKET, &index_prefix)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
restarted.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Deleted
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_partial_warehouse_index_deletion() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogObjectPaths};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-index-retry-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let index_prefix = TableCatalogObjectPaths::default().warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
assert!(indexes.len() >= 2);
|
||||
let last_index = indexes.last().unwrap();
|
||||
backend.fail_delete_attempt(RUSTFS_META_BUCKET, last_index, 1).await;
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let error = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("injected"));
|
||||
assert!(backend.list_objects(&bucket, "").await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(),
|
||||
vec![last_index.clone()]
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.list_objects(RUSTFS_META_BUCKET, &index_prefix)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_stops_before_delete_when_publication_lease_is_lost() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::test_support::RecordingLeaseClient;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogLockGuard, TableCatalogObjectPaths, TableCommitPublication};
|
||||
use rustfs_lock::LockClient;
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-lease-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 1).await;
|
||||
let index_prefix = TableCatalogObjectPaths::default().warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
TableCommitPublication::begin_table_bucket(&publication, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let client = Arc::new(RecordingLeaseClient {
|
||||
inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)))),
|
||||
acquired: std::sync::Mutex::new(None),
|
||||
});
|
||||
let lock = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"cleanup-publication-fence".to_string(),
|
||||
vec![client.clone() as Arc<dyn LockClient>],
|
||||
1,
|
||||
),
|
||||
rustfs_lock::ObjectKey::new(bucket.as_str(), "cleanup-publication-fence"),
|
||||
"cleanup-publication-owner".to_string(),
|
||||
);
|
||||
let guard = lock.get_read_lock(StdDuration::from_secs(5)).await.unwrap();
|
||||
let signal = guard.lock_lost_signal().unwrap();
|
||||
let lock_id = client.acquired.lock().unwrap().clone().unwrap();
|
||||
publication
|
||||
.publication
|
||||
.lock()
|
||||
.guards
|
||||
.push(TableCatalogLockGuard::test_namespace(guard));
|
||||
let mut cleanup = catalog
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(&bucket), &publication)
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let pause = backend.pause_next_delete(&bucket, &metadata_objects[0]).await;
|
||||
let operation = catalog.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation);
|
||||
let interfere = async {
|
||||
tokio::time::timeout(StdDuration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(client.force_release(&lock_id).await.unwrap());
|
||||
tokio::time::timeout(StdDuration::from_secs(15), signal.notified())
|
||||
.await
|
||||
.expect("cleanup publication must observe lease loss");
|
||||
pause.release();
|
||||
};
|
||||
let (result, ()) = tokio::join!(operation, interfere);
|
||||
assert!(matches!(result, Err(crate::table_catalog::TableCatalogStoreError::Unavailable(_))));
|
||||
drop(cleanup);
|
||||
TableCommitPublication::complete(&publication);
|
||||
assert!(backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert_eq!(backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(), indexes);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(!backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
@@ -76,7 +76,6 @@ pub struct AppContext {
|
||||
object_data_cache: Arc<ObjectDataCacheAdapter>,
|
||||
object_traffic_health: Arc<ObjectTrafficHealth>,
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime,
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
impl AppContext {
|
||||
@@ -128,7 +127,6 @@ impl AppContext {
|
||||
object_data_cache,
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,10 +150,6 @@ impl AppContext {
|
||||
self.table_catalog_strong_runtime.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_registration_cache(&self) -> crate::table_catalog::TableCatalogRegistrationCache {
|
||||
self.table_catalog_registration_cache.clone()
|
||||
}
|
||||
|
||||
pub fn iam(&self) -> Arc<dyn IamInterface> {
|
||||
self.iam.clone()
|
||||
}
|
||||
@@ -366,7 +360,6 @@ impl AppContext {
|
||||
object_data_cache: ObjectDataCacheAdapter::disabled_arc(),
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ use super::storage_api::multipart_usecase::contract::object::ObjectOperations as
|
||||
use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec;
|
||||
use super::storage_api::multipart_usecase::data_usage::{
|
||||
quota_object_size, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||
record_bucket_object_write_unknown_previous_memory,
|
||||
};
|
||||
use super::storage_api::multipart_usecase::error::{
|
||||
StorageError, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found,
|
||||
@@ -691,6 +690,27 @@ impl DefaultMultipartUsecase {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let current_opts = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, None, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
let object_lock_config_state = Box::pin(load_bucket_object_lock_config_state(&bucket)).await?;
|
||||
let previous_current_sizes = match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, ¤t_opts)?;
|
||||
let physical_size = existing_obj_info.size.max(0) as u64;
|
||||
let logical_size = quota_object_size(&existing_obj_info);
|
||||
Some((physical_size, logical_size))
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// A lost Complete response retries the same upload id after staging is
|
||||
// gone. `get_multipart_info` is NoSuchUpload in that case; completion
|
||||
// below replays the committed object instead of failing the retry.
|
||||
@@ -720,6 +740,11 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
validate_complete_multipart_checksum_type(&req.headers, &info.user_defined)?;
|
||||
}
|
||||
let cache_adapter = self.object_data_cache();
|
||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
||||
|
||||
let upload_user_defined = multipart_info.as_ref().map(|info| info.user_defined.clone());
|
||||
let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&req);
|
||||
|
||||
let quota_metadata_sys = self.bucket_metadata_sys();
|
||||
let quota_tracking = quota_metadata_sys.is_some();
|
||||
@@ -732,42 +757,13 @@ impl DefaultMultipartUsecase {
|
||||
apply_quota_admission(&mut opts, &check_result)?;
|
||||
}
|
||||
|
||||
let object_lock_config_state = Box::pin(load_bucket_object_lock_config_state(&bucket)).await?;
|
||||
let prelookup_required = quota_enabled
|
||||
|| (!super::object::shared::put_like_write_creates_new_version(&opts)
|
||||
&& super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state));
|
||||
let prelookup_previous_current_size: Option<Option<u64>> = if prelookup_required {
|
||||
let current_opts = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, None, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, ¤t_opts)?;
|
||||
Some(Some(if quota_enabled {
|
||||
quota_object_size(&existing_obj_info).map_err(ApiError::from)?
|
||||
} else {
|
||||
existing_obj_info.size.max(0) as u64
|
||||
}))
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
Some(None)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
let previous_current_size = match previous_current_sizes {
|
||||
Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size),
|
||||
Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err).into()),
|
||||
Some((physical_size, _)) => Some(physical_size),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let cache_adapter = self.object_data_cache();
|
||||
let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
|
||||
|
||||
let upload_user_defined = multipart_info.as_ref().map(|info| info.user_defined.clone());
|
||||
let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&req);
|
||||
|
||||
// Replication admission is decided at the object-creation commit, not
|
||||
// at multipart-session creation. Configuration and matching targets
|
||||
// can change while parts are uploaded, so persist the exact decision's
|
||||
@@ -835,20 +831,11 @@ impl DefaultMultipartUsecase {
|
||||
if !obj_info.multipart_completion_replayed {
|
||||
if quota_tracking {
|
||||
let committed_size = quota_accounting_object_size(&obj_info, quota_enabled)?;
|
||||
match prelookup_previous_current_size
|
||||
.or_else(|| super::object::previous_current_size_from_backfill(obj_info.backfilled_old_current_size))
|
||||
{
|
||||
Some(previous_current_size) => {
|
||||
if versioned {
|
||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size)
|
||||
.await;
|
||||
} else {
|
||||
record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, versioned).await;
|
||||
}
|
||||
|
||||
if versioned {
|
||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||
} else {
|
||||
record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1172,22 +1159,18 @@ impl DefaultMultipartUsecase {
|
||||
);
|
||||
}
|
||||
|
||||
if !super::object::shared::put_like_write_creates_new_version(&opts)
|
||||
&& super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state)
|
||||
{
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,10 +230,7 @@ pub(crate) use self::on_demand_migration_put::*;
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::put::PollCountingBody;
|
||||
use self::put::*;
|
||||
pub(crate) use self::put::{
|
||||
guard_put_object_body_read_timeout, previous_current_size_from_backfill, put_object_body_read_timeout,
|
||||
reject_oversize_single_upload,
|
||||
};
|
||||
pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout, reject_oversize_single_upload};
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::restore::RestoreStatusCommitBarrier;
|
||||
pub(crate) use self::shared::*;
|
||||
|
||||
@@ -2324,7 +2324,7 @@ impl DefaultObjectUsecase {
|
||||
/// `previous_current_size` value the usage-accounting helpers expect. Outer
|
||||
/// `None` = unknown (no quorum agreement, or a peer predates the field) — the
|
||||
/// caller must fall back to the degraded accounting path.
|
||||
pub(crate) fn previous_current_size_from_backfill(backfill: Option<OldCurrentSize>) -> Option<Option<u64>> {
|
||||
pub(super) fn previous_current_size_from_backfill(backfill: Option<OldCurrentSize>) -> Option<Option<u64>> {
|
||||
backfill.map(|observation| match observation {
|
||||
OldCurrentSize::Present(size) => Some(size.max(0) as u64),
|
||||
OldCurrentSize::Absent => None,
|
||||
|
||||
@@ -874,7 +874,7 @@ pub(crate) fn build_put_like_object_lock_metadata(
|
||||
Ok(Some(eval_metadata))
|
||||
}
|
||||
|
||||
pub(in crate::app) fn put_like_write_creates_new_version(opts: &ObjectOptions) -> bool {
|
||||
pub(super) fn put_like_write_creates_new_version(opts: &ObjectOptions) -> bool {
|
||||
opts.version_id.is_none() && opts.versioned && !opts.version_suspended
|
||||
}
|
||||
|
||||
@@ -997,7 +997,7 @@ pub(crate) async fn object_lock_checks_required(bucket: &str) -> bool {
|
||||
.map_or(true, |metadata| metadata.object_lock_checks_required())
|
||||
}
|
||||
|
||||
pub(in crate::app) fn object_lock_checks_required_for_state(state: &metadata_sys::ObjectLockConfigState) -> bool {
|
||||
pub(super) fn object_lock_checks_required_for_state(state: &metadata_sys::ObjectLockConfigState) -> bool {
|
||||
match state {
|
||||
metadata_sys::ObjectLockConfigState::Configured { .. } | metadata_sys::ObjectLockConfigState::Fabricated => true,
|
||||
metadata_sys::ObjectLockConfigState::ConfirmedAbsent => false,
|
||||
|
||||
@@ -490,8 +490,7 @@ pub(crate) mod table {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::storage_contracts::{
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr,
|
||||
WalkOptions,
|
||||
ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr, WalkOptions,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -499,10 +498,6 @@ pub(crate) mod table {
|
||||
pub(crate) use super::super::super::storage_contracts::NamespaceLocking;
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use super::super::super::storage_contracts::MultipartUploadInspection;
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
@@ -1,342 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Validates one metadata object; callers must also validate the complete object inventory.
|
||||
pub(crate) fn validate_empty_table_cleanup_metadata(
|
||||
table_bucket: &str,
|
||||
object_key: &str,
|
||||
metadata: &[u8],
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let (namespace, table) = table_identity_from_metadata_object_key(object_key).ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid("cleanup metadata must use the default table metadata directory".to_string())
|
||||
})?;
|
||||
let metadata = decode_table_metadata_json(object_key, metadata)?;
|
||||
validate_supported_table_metadata(&metadata)?;
|
||||
|
||||
// Future metadata fields may introduce additional object references.
|
||||
for field in metadata.as_object().expect("validated metadata is a JSON object").keys() {
|
||||
if !matches!(
|
||||
field.as_str(),
|
||||
"format-version"
|
||||
| "table-uuid"
|
||||
| "location"
|
||||
| "last-sequence-number"
|
||||
| "last-updated-ms"
|
||||
| "last-column-id"
|
||||
| "schema"
|
||||
| "schemas"
|
||||
| "current-schema-id"
|
||||
| "partition-spec"
|
||||
| "partition-specs"
|
||||
| "default-spec-id"
|
||||
| "last-partition-id"
|
||||
| "properties"
|
||||
| "current-snapshot-id"
|
||||
| "snapshots"
|
||||
| "snapshot-log"
|
||||
| "metadata-log"
|
||||
| "sort-orders"
|
||||
| "default-sort-order-id"
|
||||
| "refs"
|
||||
| "statistics"
|
||||
| "partition-statistics"
|
||||
) {
|
||||
return Err(TableCatalogStoreError::Unsupported(format!(
|
||||
"empty table cleanup does not support metadata field {field}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] {
|
||||
let Some(value) = metadata.get(field) else {
|
||||
continue;
|
||||
};
|
||||
let entries = value
|
||||
.as_array()
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid(format!("cleanup metadata field {field} must be an array")))?;
|
||||
if !entries.is_empty() {
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"empty table cleanup requires metadata field {field} to be empty"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Some(value) = metadata.get("refs") {
|
||||
let refs = value
|
||||
.as_object()
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid("cleanup metadata refs must be an object".to_string()))?;
|
||||
if !refs.is_empty() {
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"empty table cleanup requires metadata refs to be empty".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let table_uuid = table_metadata_uuid(&metadata)?;
|
||||
if Uuid::parse_str(table_uuid).ok().filter(|uuid| !uuid.is_nil()).is_none() {
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"cleanup metadata table-uuid must be a non-nil UUID".to_string(),
|
||||
));
|
||||
}
|
||||
let warehouse_prefix = table_warehouse_object_prefix_from_location(table_bucket, table_metadata_location(&metadata)?)?;
|
||||
if warehouse_prefix
|
||||
.strip_prefix("tables/")
|
||||
.and_then(|prefix| prefix.strip_suffix('/'))
|
||||
.and_then(|table_id| Uuid::parse_str(table_id).ok())
|
||||
.filter(|table_id| !table_id.is_nil())
|
||||
.is_none()
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"empty table cleanup requires the default tables/<table-id> warehouse location".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(entries) = metadata.get("metadata-log").and_then(serde_json::Value::as_array) {
|
||||
for entry in entries {
|
||||
let entry = entry.as_object().expect("validated metadata log entries are JSON objects");
|
||||
if entry
|
||||
.keys()
|
||||
.any(|field| !matches!(field.as_str(), "timestamp-ms" | "metadata-file"))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table cleanup does not support unknown metadata-log fields".to_string(),
|
||||
));
|
||||
}
|
||||
let location = entry["metadata-file"]
|
||||
.as_str()
|
||||
.expect("validated metadata log entries contain metadata-file");
|
||||
table_catalog_object_key_from_location(table_bucket, location)
|
||||
.filter(|location| is_valid_table_metadata_location(&namespace, &table, location))
|
||||
.filter(|location| location != object_key)
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid(
|
||||
"cleanup metadata-log must reference another file in the same table metadata directory".to_string(),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::table_metadata_json;
|
||||
use serde_json::{Value, json};
|
||||
use std::io::Write;
|
||||
|
||||
const BUCKET: &str = "warehouse";
|
||||
const TABLE_UUID: &str = "f6dd46c2-4971-4d41-b188-452d5f5c2c01";
|
||||
const WAREHOUSE_LOCATION: &str = "s3://warehouse/tables/a9d8f3b1-abde-4a0b-b6fb-e36a30abff16";
|
||||
const OBJECT: &str =
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/events/tables/orders/metadata/00002.metadata.json";
|
||||
const PREVIOUS: &str =
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/events/tables/orders/metadata/00001.metadata.json";
|
||||
|
||||
fn empty_metadata() -> Value {
|
||||
table_metadata_json(TABLE_UUID, WAREHOUSE_LOCATION)
|
||||
}
|
||||
|
||||
fn validate(metadata: &Value) -> TableCatalogStoreResult<()> {
|
||||
validate_empty_table_cleanup_metadata(BUCKET, OBJECT, &serde_json::to_vec(metadata).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_v2_and_same_directory_history() {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([
|
||||
{"timestamp-ms": 1, "metadata-file": PREVIOUS},
|
||||
{"timestamp-ms": 1, "metadata-file": format!("s3://{BUCKET}/{PREVIOUS}")}
|
||||
]);
|
||||
assert!(validate(&metadata).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_v1_without_snapshots() {
|
||||
let mut metadata = json!({
|
||||
"format-version": 1,
|
||||
"table-uuid": TABLE_UUID,
|
||||
"location": WAREHOUSE_LOCATION,
|
||||
"last-updated-ms": 1,
|
||||
"last-column-id": 0,
|
||||
"schema": {"type": "struct", "fields": []},
|
||||
"partition-spec": [],
|
||||
"current-snapshot-id": -1
|
||||
});
|
||||
assert!(validate(&metadata).is_ok());
|
||||
metadata["snapshots"] = json!([{
|
||||
"snapshot-id": 1,
|
||||
"timestamp-ms": 1,
|
||||
"manifests": ["s3://warehouse/tables/table-id/metadata/manifest.avro"]
|
||||
}]);
|
||||
validate_supported_table_metadata(&metadata).unwrap();
|
||||
assert!(matches!(validate(&metadata), Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_unsupported_versions_and_unknown_references() {
|
||||
for version in [0, 3, 4] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["format-version"] = json!(version);
|
||||
assert!(validate(&metadata).is_err(), "version {version}");
|
||||
}
|
||||
for field in [
|
||||
"manifest-list",
|
||||
"manifests",
|
||||
"data-files",
|
||||
"delete-files",
|
||||
"next-row-id",
|
||||
"future-field",
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata[field] = json!([]);
|
||||
assert!(validate(&metadata).is_err(), "field {field}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_snapshot_and_history_references() {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["snapshots"] = json!([{
|
||||
"snapshot-id": 1,
|
||||
"sequence-number": 0,
|
||||
"timestamp-ms": 1,
|
||||
"manifest-list": "s3://warehouse/tables/table-id/metadata/list.avro",
|
||||
"summary": {"operation": "append"}
|
||||
}]);
|
||||
validate_supported_table_metadata(&metadata).unwrap();
|
||||
assert!(matches!(validate(&metadata), Err(TableCatalogStoreError::Conflict(_))));
|
||||
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["snapshot-log"] = json!([{"timestamp-ms": 1, "snapshot-id": 1}]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
metadata = empty_metadata();
|
||||
metadata["refs"] = json!({"main": {"snapshot-id": 1, "type": "branch"}});
|
||||
assert!(validate(&metadata).is_err());
|
||||
metadata = empty_metadata();
|
||||
metadata["current-snapshot-id"] = json!(1);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_only_absent_null_or_minus_one_current_snapshot() {
|
||||
for version in [1, 2] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["format-version"] = json!(version);
|
||||
if version == 1 {
|
||||
metadata["schema"] = metadata["schemas"][0].clone();
|
||||
metadata["partition-spec"] = json!([]);
|
||||
}
|
||||
assert!(validate(&metadata).is_ok());
|
||||
for value in [Value::Null, json!(-1)] {
|
||||
metadata["current-snapshot-id"] = value;
|
||||
assert!(validate(&metadata).is_ok(), "version {version}");
|
||||
}
|
||||
for value in [json!(1), json!(0), json!(-2), json!("1"), json!(false), json!([])] {
|
||||
metadata["current-snapshot-id"] = value;
|
||||
assert!(validate(&metadata).is_err(), "version {version}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_statistics_references_and_malformed_empty_fields() {
|
||||
for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] {
|
||||
for value in [Value::Null, json!({}), json!("empty"), json!([{}])] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata[field] = value;
|
||||
assert!(validate(&metadata).is_err(), "field {field}");
|
||||
}
|
||||
}
|
||||
for value in [Value::Null, json!([]), json!("empty")] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["refs"] = value;
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_unknown_or_cross_table_history() {
|
||||
for location in [
|
||||
format!("s3://other/{PREVIOUS}"),
|
||||
PREVIOUS.replace("/orders/", "/other/"),
|
||||
PREVIOUS.replace("/analytics/events/", "/another/"),
|
||||
PREVIOUS.replace("00001.metadata.json", "../00001.metadata.json"),
|
||||
PREVIOUS.replace("00001.metadata.json", "%2e%2e.json"),
|
||||
format!("{PREVIOUS}?versionId=previous"),
|
||||
format!("/{PREVIOUS}"),
|
||||
OBJECT.to_string(),
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([{"timestamp-ms": 1, "metadata-file": location}]);
|
||||
assert!(validate(&metadata).is_err(), "location {location}");
|
||||
}
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([{"timestamp-ms": 1, "metadata-file": PREVIOUS, "future-file": "data.parquet"}]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_requires_valid_schema_uuid_and_warehouse() {
|
||||
for uuid in ["table-uuid", "00000000-0000-0000-0000-000000000000", ""] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["table-uuid"] = json!(uuid);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
for location in [
|
||||
"s3://other/tables/table-id",
|
||||
"s3://warehouse/",
|
||||
"s3://warehouse/.rustfs-table/metadata",
|
||||
"file:///table",
|
||||
"s3://warehouse/custom/table-id",
|
||||
"s3://warehouse/tables/table-id",
|
||||
"s3://warehouse/tables/00000000-0000-0000-0000-000000000000",
|
||||
"s3://warehouse/tables/a9d8f3b1-abde-4a0b-b6fb-e36a30abff16/nested",
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["location"] = json!(location);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["schemas"] = json!([]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_non_default_metadata_paths() {
|
||||
let bytes = serde_json::to_vec(&empty_metadata()).unwrap();
|
||||
for object in [
|
||||
OBJECT.replace("/default/", "/custom/"),
|
||||
OBJECT.replace("/tables/", "/views/"),
|
||||
OBJECT.replace("/metadata/", "/data/"),
|
||||
OBJECT.replace("00002.metadata.json", "part.parquet"),
|
||||
OBJECT.replace("00002.metadata.json", "nested/00002.metadata.json"),
|
||||
format!("s3://{BUCKET}/{OBJECT}"),
|
||||
] {
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, &bytes).is_err(), "object {object}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_decodes_gzip_and_rejects_invalid_json() {
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(&serde_json::to_vec(&empty_metadata()).unwrap()).unwrap();
|
||||
let bytes = encoder.finish().unwrap();
|
||||
let object = OBJECT.replace(".metadata.json", ".gz.metadata.json");
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, &bytes).is_ok());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, OBJECT, b"{}").is_err());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, OBJECT, b"[").is_err());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, b"invalid gzip").is_err());
|
||||
}
|
||||
}
|
||||
@@ -28,11 +28,9 @@ use std::{
|
||||
|
||||
use crate::storage_api::table::contract::http::HTTPPreconditions;
|
||||
use crate::storage_api::table::contract::list::{
|
||||
AuthoritativeListOperations as StorageAuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as StorageListOperations,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use crate::storage_api::table::contract::multipart::MultipartUploadInspection as StorageMultipartUploadInspection;
|
||||
use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking;
|
||||
use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations};
|
||||
use crate::storage_api::table::contract::range::HTTPRangeSpec;
|
||||
@@ -60,7 +58,6 @@ use crate::storage_api::table::{
|
||||
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
|
||||
mod cleanup_metadata;
|
||||
mod error;
|
||||
mod iceberg;
|
||||
mod identifier;
|
||||
@@ -249,9 +246,7 @@ pub(crate) trait TableCatalogStorage:
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageAuthoritativeListOperations
|
||||
+ StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ StorageMultipartUploadInspection<Error = EcstoreError>
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
{
|
||||
}
|
||||
|
||||
@@ -279,9 +274,7 @@ impl<T> TableCatalogStorage for T where
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageAuthoritativeListOperations
|
||||
+ StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ StorageMultipartUploadInspection<Error = EcstoreError>
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::object::{
|
||||
next_table_catalog_update_time, table_warehouse_index_state_ready, validate_table_warehouse_index_entry_object,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
pub(super) struct TableBucketDisablement {
|
||||
pub(super) entry: TableBucketEntry,
|
||||
guards: Vec<TableCatalogLockGuard>,
|
||||
}
|
||||
|
||||
/// An in-memory, fenced cleanup plan. No mutation occurs until every object has
|
||||
/// been validated and the caller has authorized all metadata deletions.
|
||||
pub(crate) struct EmptyTableBucketCleanup {
|
||||
disablement: TableBucketDisablement,
|
||||
metadata: Vec<(String, String)>,
|
||||
indexes: Vec<(String, String)>,
|
||||
fence: TableCatalogObjectMutationFence,
|
||||
}
|
||||
|
||||
impl EmptyTableBucketCleanup {
|
||||
pub(crate) fn metadata_objects(&self) -> impl Iterator<Item = &str> {
|
||||
self.metadata.iter().map(|(key, _)| key.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_held(&self) -> TableCatalogStoreResult<()> {
|
||||
self.fence.ensure_held()
|
||||
}
|
||||
|
||||
pub(crate) fn mutation_fence(&self) -> TableCatalogObjectMutationFence {
|
||||
self.fence.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_conflict(message: impl Into<String>) -> TableCatalogStoreError {
|
||||
TableCatalogStoreError::Conflict(message.into())
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
|
||||
pub(super) async fn begin_table_bucket_disablement(
|
||||
&self,
|
||||
mut entry: TableBucketEntry,
|
||||
publication: Option<&dyn TableCommitPublication>,
|
||||
) -> TableCatalogStoreResult<TableBucketDisablement> {
|
||||
validate_table_bucket_entry(&entry)?;
|
||||
let mut guards = Vec::new();
|
||||
if publication.is_some_and(|publication| publication.catalog_migration_read_permit_status() == Some(true)) {
|
||||
if self
|
||||
.backend
|
||||
.read_object(self.catalog_bucket(), &self.paths.backing_migration_global_fence_path())
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket registry migration is in progress"));
|
||||
}
|
||||
self.ensure_object_backed_catalog_write_permit_after_lock(&entry.table_bucket)
|
||||
.await?;
|
||||
} else {
|
||||
guards.push(self.acquire_table_bucket_registry_write_permit().await?);
|
||||
guards.push(self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?);
|
||||
}
|
||||
let object = self.paths.table_bucket_entry_path(&entry.table_bucket);
|
||||
guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?);
|
||||
if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? {
|
||||
if current.active_rename_id.is_some()
|
||||
|| !matches!(current.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted)
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket has an operation in progress"));
|
||||
}
|
||||
entry = current;
|
||||
}
|
||||
Ok(TableBucketDisablement { entry, guards })
|
||||
}
|
||||
|
||||
pub(super) async fn finish_table_bucket_disablement(
|
||||
&self,
|
||||
disablement: &mut TableBucketDisablement,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut entry = disablement.entry.clone();
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
entry.updated_at = Some(next_table_catalog_update_time(entry.updated_at.as_deref()));
|
||||
self.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&self.paths.table_bucket_entry_path(&entry.table_bucket),
|
||||
&entry,
|
||||
TableCatalogPutPrecondition::Any,
|
||||
disablement
|
||||
.guards
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
entry: TableBucketEntry,
|
||||
publication: &impl TableCatalogObjectBackend,
|
||||
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(cleanup_conflict("empty table bucket cleanup requires a publication fence"));
|
||||
}
|
||||
let disablement = self.begin_table_bucket_disablement(entry, Some(publication)).await?;
|
||||
let bucket = &disablement.entry.table_bucket;
|
||||
let mut fence = publication
|
||||
.object_mutation_fence()
|
||||
.ok_or_else(|| cleanup_conflict("cleanup requires storage fencing"))?;
|
||||
fence
|
||||
.signals
|
||||
.extend(disablement.guards.iter().filter_map(TableCatalogLockGuard::lock_lost_signal));
|
||||
fence.ensure_held()?;
|
||||
|
||||
let catalog_objects = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &self.paths.table_bucket_root_prefix(bucket))
|
||||
.await?;
|
||||
let bucket_path = self.paths.table_bucket_entry_path(bucket);
|
||||
let index_state_path = self.paths.warehouse_index_state_path(bucket);
|
||||
let index_prefix = self.paths.warehouse_index_entries_prefix(bucket);
|
||||
let mut indexes = Vec::new();
|
||||
for object in catalog_objects {
|
||||
if object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
{
|
||||
return Err(cleanup_conflict("catalog contains versioned or deleted records"));
|
||||
}
|
||||
if object.key == bucket_path {
|
||||
continue;
|
||||
}
|
||||
let etag = if object.key == index_state_path {
|
||||
let (state, etag) = self
|
||||
.read_entry::<TableWarehouseIndexStateEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("warehouse index state changed during cleanup"))?;
|
||||
table_warehouse_index_state_ready(&state, bucket)?;
|
||||
if !matches!(state.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted) {
|
||||
return Err(cleanup_conflict("warehouse index has an operation in progress"));
|
||||
}
|
||||
etag
|
||||
} else if object.key.starts_with(&index_prefix) {
|
||||
let (index, etag) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("warehouse index changed during cleanup"))?;
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object.key, &index)?;
|
||||
if index.table_bucket != *bucket || index.state != TableCatalogEntryState::Deleted {
|
||||
return Err(cleanup_conflict("table bucket contains an active warehouse index"));
|
||||
}
|
||||
Namespace::parse(&index.namespace).map_err(|err| cleanup_conflict(err.to_string()))?;
|
||||
IdentifierSegment::parse(index.table.clone()).map_err(|err| cleanup_conflict(err.to_string()))?;
|
||||
if Uuid::parse_str(&index.table_id).ok().filter(|id| !id.is_nil()).is_none() {
|
||||
return Err(cleanup_conflict("warehouse index table identity is invalid"));
|
||||
}
|
||||
let expected = format!("tables/{}/", index.table_id);
|
||||
if index.warehouse_object_prefix != expected {
|
||||
return Err(cleanup_conflict("cleanup requires the default table warehouse location"));
|
||||
}
|
||||
etag
|
||||
} else {
|
||||
return Err(cleanup_conflict("table bucket contains namespaces or retained catalog resources"));
|
||||
};
|
||||
let etag = etag.ok_or_else(|| cleanup_conflict("catalog cleanup object has no etag"))?;
|
||||
indexes.push((object.key, etag));
|
||||
}
|
||||
self.backend.ensure_no_multipart_uploads(bucket).await?;
|
||||
let mut metadata = Vec::new();
|
||||
for object in self.backend.list_objects_for_cleanup(bucket, "").await? {
|
||||
if object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket contains object versions or delete markers"));
|
||||
}
|
||||
let value = publication
|
||||
.read_object_limited(bucket, &object.key, TABLE_METADATA_JSON_MAX_SIZE)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("metadata object changed during cleanup"))?;
|
||||
crate::table_catalog::cleanup_metadata::validate_empty_table_cleanup_metadata(bucket, &object.key, &value.data)?;
|
||||
let etag = value
|
||||
.etag
|
||||
.ok_or_else(|| cleanup_conflict("metadata cleanup object has no etag"))?;
|
||||
metadata.push((object.key, etag));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
Ok(EmptyTableBucketCleanup {
|
||||
disablement,
|
||||
metadata,
|
||||
indexes,
|
||||
fence,
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
cleanup: &mut EmptyTableBucketCleanup,
|
||||
incarnation: Uuid,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let bucket = &cleanup.disablement.entry.table_bucket;
|
||||
for (key, etag) in &cleanup.metadata {
|
||||
cleanup.ensure_held()?;
|
||||
self.backend
|
||||
.delete_object_if_match_fenced_for_incarnation(bucket, key, etag, incarnation, &cleanup.fence)
|
||||
.await?;
|
||||
}
|
||||
// A retry inventories the remaining metadata. Keep warehouse protection
|
||||
// until all user objects and uploads have been checked again.
|
||||
if !self.backend.list_objects_for_cleanup(bucket, "").await?.is_empty() {
|
||||
return Err(cleanup_conflict("table bucket changed during cleanup"));
|
||||
}
|
||||
self.backend.ensure_no_multipart_uploads(bucket).await?;
|
||||
for (key, etag) in &cleanup.indexes {
|
||||
cleanup.ensure_held()?;
|
||||
self.backend
|
||||
.delete_object_if_match_fenced(self.catalog_bucket(), key, etag, &cleanup.fence)
|
||||
.await?;
|
||||
}
|
||||
cleanup.ensure_held()?;
|
||||
self.finish_table_bucket_disablement(&mut cleanup.disablement).await
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ConfiguredTableCatalogStore<B> {
|
||||
pub(crate) async fn prepare_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
entry: TableBucketEntry,
|
||||
publication: &impl TableCatalogObjectBackend,
|
||||
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.prepare_empty_table_bucket_cleanup(entry, publication).await,
|
||||
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
cleanup: &mut EmptyTableBucketCleanup,
|
||||
incarnation: Uuid,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.execute_empty_table_bucket_cleanup(cleanup, incarnation).await,
|
||||
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,13 +17,8 @@ use super::*;
|
||||
use crate::storage_api::table::{WriteCommitGuard, WriteCompletion};
|
||||
|
||||
mod backup;
|
||||
mod cleanup;
|
||||
mod migration;
|
||||
mod object;
|
||||
mod registration_cache;
|
||||
#[cfg(test)]
|
||||
mod registration_cache_tests;
|
||||
pub(crate) use registration_cache::TableCatalogRegistrationCache;
|
||||
mod strong;
|
||||
|
||||
pub(crate) use backup::{TableCatalogBackupReport, TableCatalogRestoreReport};
|
||||
@@ -780,14 +775,6 @@ pub(crate) struct TableCatalogObjectListPage {
|
||||
pub is_truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct TableCatalogCleanupObject {
|
||||
pub key: String,
|
||||
pub version_id: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
pub delete_marker: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum TableCatalogPutPrecondition {
|
||||
Any,
|
||||
@@ -816,10 +803,6 @@ pub(in crate::table_catalog) fn catalog_list_next_continuation(
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
None
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
None
|
||||
}
|
||||
@@ -935,30 +918,6 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn put_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
_write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
self.put_object_fenced(bucket, object, data, precondition, fence).await
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_unlocked(bucket, object).await?;
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()>;
|
||||
|
||||
async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()>;
|
||||
@@ -975,41 +934,12 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_expected_etag: &str,
|
||||
_expected_incarnation_id: Uuid,
|
||||
_fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"incarnation-checked cleanup is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object(bucket, object).await
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>>;
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"authoritative cleanup listing is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, _bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"multipart cleanup validation is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_objects_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2274,7 +2204,6 @@ where
|
||||
pub(crate) struct EcStoreTableCatalogObjectBackend<S> {
|
||||
store: Arc<S>,
|
||||
strong_runtime: StrongTableCatalogRuntime,
|
||||
registration_cache: TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
@@ -2282,7 +2211,6 @@ impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
Self {
|
||||
store: self.store.clone(),
|
||||
strong_runtime: self.strong_runtime.clone(),
|
||||
registration_cache: self.registration_cache.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2292,19 +2220,7 @@ where
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
pub fn new_with_strong_runtime(store: Arc<S>, strong_runtime: StrongTableCatalogRuntime) -> Self {
|
||||
Self::new_with_runtimes(store, strong_runtime, TableCatalogRegistrationCache::default())
|
||||
}
|
||||
|
||||
pub fn new_with_runtimes(
|
||||
store: Arc<S>,
|
||||
strong_runtime: StrongTableCatalogRuntime,
|
||||
registration_cache: TableCatalogRegistrationCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
strong_runtime,
|
||||
registration_cache,
|
||||
}
|
||||
Self { store, strong_runtime }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2315,10 +2231,6 @@ impl<S> TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend<S>
|
||||
where
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
Some(self.registration_cache.clone())
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
Some(self.strong_runtime.clone())
|
||||
}
|
||||
@@ -2478,46 +2390,6 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.put_object_with_options(bucket, object, data, precondition, Some(write_guards), Some(fence))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
if write_guards.is_empty() {
|
||||
return Err(TableCatalogStoreError::Internal("catalog delete has no namespace owner".to_string()));
|
||||
}
|
||||
let mut options = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_object_options(&mut options);
|
||||
for guard in &write_guards {
|
||||
options.add_write_commit_guard(guard);
|
||||
}
|
||||
match self.store.delete_object(bucket, object, options).await {
|
||||
Ok(_) => fence.ensure_held(),
|
||||
Err(err) if is_missing_storage_error(&err) => fence.ensure_held(),
|
||||
Err(err) => Err(storage_error_to_catalog("delete fenced catalog object", err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_object_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2540,7 +2412,7 @@ where
|
||||
}
|
||||
|
||||
async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, None, None)
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -2552,25 +2424,7 @@ where
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence), None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if expected_incarnation_id.is_nil() {
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"cleanup requires a non-nil bucket incarnation".to_string(),
|
||||
));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence), Some(expected_incarnation_id))
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -2644,57 +2498,6 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel::<CatalogObjectInfoOrErr>(100);
|
||||
let walk = self.store.clone().walk_authoritative(
|
||||
cancel,
|
||||
bucket,
|
||||
prefix,
|
||||
sender,
|
||||
CatalogWalkOptions {
|
||||
include_free_versions: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let collect = async move {
|
||||
let mut objects = Vec::new();
|
||||
while let Some(result) = receiver.recv().await {
|
||||
if let Some(err) = result.err {
|
||||
return Err(storage_error_to_catalog("inspect cleanup object versions", err));
|
||||
}
|
||||
let object = result.item.ok_or_else(|| {
|
||||
TableCatalogStoreError::Internal("cleanup object walk returned an empty result".to_string())
|
||||
})?;
|
||||
objects.push(TableCatalogCleanupObject {
|
||||
key: object.name,
|
||||
version_id: object.version_id.filter(|id| !id.is_nil()).map(|id| id.to_string()),
|
||||
etag: object.etag,
|
||||
delete_marker: object.delete_marker,
|
||||
});
|
||||
}
|
||||
Ok(objects)
|
||||
};
|
||||
let (walk_result, objects) = tokio::join!(walk, collect);
|
||||
let objects = objects?;
|
||||
walk_result.map_err(|err| storage_error_to_catalog("list cleanup object versions", err))?;
|
||||
Ok(objects)
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
self.store.ensure_no_multipart_uploads(bucket).await.map_err(|err| match err {
|
||||
StorageError::BucketNotEmpty(_) => {
|
||||
TableCatalogStoreError::Conflict(format!("table bucket {bucket} contains unfinished multipart uploads"))
|
||||
}
|
||||
other => storage_error_to_catalog("inspect unfinished multipart uploads", other),
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<TableCatalogLockGuard> {
|
||||
let lock = self
|
||||
.store
|
||||
@@ -2731,14 +2534,12 @@ where
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut options = ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(expected_etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
expected_bucket_incarnation_id: expected_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(fence) = fence {
|
||||
|
||||
@@ -13,9 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
|
||||
pub(super) const TABLE_REGISTRATION_READ_CONCURRENCY: usize = 8;
|
||||
const TABLE_MAINTENANCE_RENAME_MAX_OBJECTS: usize = 4096;
|
||||
const TABLE_MAINTENANCE_SCHEDULER_HISTORY_MAX_OBJECTS: usize = 4096;
|
||||
|
||||
@@ -263,7 +261,7 @@ fn validate_table_rename_intent_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn next_table_catalog_update_time(previous: Option<&str>) -> String {
|
||||
fn next_table_catalog_update_time(previous: Option<&str>) -> String {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let update_time = previous
|
||||
.and_then(|value| OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).ok())
|
||||
@@ -362,7 +360,7 @@ pub(super) fn validate_table_maintenance_report_owner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_table_warehouse_index_entry_object(
|
||||
fn validate_table_warehouse_index_entry_object(
|
||||
paths: &TableCatalogObjectPaths,
|
||||
object: &str,
|
||||
index: &TableWarehouseIndexEntry,
|
||||
@@ -376,10 +374,7 @@ pub(super) fn validate_table_warehouse_index_entry_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn table_warehouse_index_state_ready(
|
||||
state: &TableWarehouseIndexStateEntry,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<bool> {
|
||||
fn table_warehouse_index_state_ready(state: &TableWarehouseIndexStateEntry, table_bucket: &str) -> TableCatalogStoreResult<bool> {
|
||||
if state.version == 0 || state.version > TABLE_WAREHOUSE_INDEX_STATE_VERSION {
|
||||
return Err(TableCatalogStoreError::Invalid(format!(
|
||||
"unsupported warehouse index state version: {}",
|
||||
@@ -1222,7 +1217,7 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn write_entry_unlocked<T>(
|
||||
async fn write_entry_unlocked<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1230,39 +1225,14 @@ where
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
) -> TableCatalogStoreResult<()>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
self.write_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_entry_unlocked_with_fence<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
entry: &T,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
) -> TableCatalogStoreResult<()>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
let data = serde_json::to_vec(entry)
|
||||
.map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?;
|
||||
match fence {
|
||||
Some(fence) => {
|
||||
self.backend
|
||||
.put_object_unlocked_fenced(bucket, object, data, precondition, write_guards, fence)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.backend
|
||||
.put_object_unlocked(bucket, object, data, precondition, write_guards)
|
||||
.await
|
||||
}
|
||||
}
|
||||
self.backend
|
||||
.put_object_unlocked(bucket, object, data, precondition, write_guards)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_exact_entry_unlocked<T>(
|
||||
@@ -1273,27 +1243,11 @@ where
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
) -> TableCatalogStoreResult<String>
|
||||
where
|
||||
T: DeserializeOwned + PartialEq + Serialize,
|
||||
{
|
||||
self.write_exact_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_exact_entry_unlocked_with_fence<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
entry: &T,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
) -> TableCatalogStoreResult<String>
|
||||
where
|
||||
T: DeserializeOwned + PartialEq + Serialize,
|
||||
{
|
||||
let write_result = self
|
||||
.write_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, fence)
|
||||
.write_entry_unlocked(bucket, object, entry, precondition, write_guards)
|
||||
.await;
|
||||
let current = self.read_entry_unlocked::<T>(bucket, object).await?;
|
||||
match current {
|
||||
@@ -1310,7 +1264,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn read_table_bucket_with_etag_unlocked(
|
||||
async fn read_table_bucket_with_etag_unlocked(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<Option<(TableBucketEntry, String)>> {
|
||||
@@ -1433,7 +1387,6 @@ where
|
||||
"table rename recovery requires a table-bucket publication fence".to_string(),
|
||||
));
|
||||
}
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let bucket_object = self.paths.table_bucket_entry_path(table_bucket);
|
||||
let Some((observed_bucket, _)) = self
|
||||
.read_entry::<TableBucketEntry>(self.catalog_bucket(), &bucket_object)
|
||||
@@ -1558,7 +1511,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == source_fence || current == source_tombstone => {}
|
||||
Some((current, current_etag)) if current == intent.source && current_etag == intent.source_etag => {
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&source_object,
|
||||
&source_fence,
|
||||
@@ -1567,7 +1520,6 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1597,7 +1549,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == destination_fence || current == intent.destination => {}
|
||||
None if intent.destination_etag.is_none() => {
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&destination_fence,
|
||||
@@ -1606,7 +1558,6 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1614,7 +1565,7 @@ where
|
||||
if current.state == TableCatalogEntryState::Deleted
|
||||
&& Some(current_etag.as_str()) == intent.destination_etag.as_deref() =>
|
||||
{
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&destination_fence,
|
||||
@@ -1623,7 +1574,6 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1651,7 +1601,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == source_tombstone => {}
|
||||
Some((current, current_etag)) if current == source_fence => {
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&source_object,
|
||||
&source_tombstone,
|
||||
@@ -1660,7 +1610,6 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1698,7 +1647,7 @@ where
|
||||
"table rename warehouse index changed during recovery".to_string(),
|
||||
));
|
||||
}
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&index_object,
|
||||
&destination_index,
|
||||
@@ -1707,7 +1656,6 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1726,7 +1674,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == intent.destination => {}
|
||||
Some((current, current_etag)) if current == destination_fence => {
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&intent.destination,
|
||||
@@ -1735,7 +1683,6 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1773,13 +1720,12 @@ where
|
||||
}
|
||||
bucket_entry.active_rename_id = None;
|
||||
bucket_entry.updated_at = Some(next_table_catalog_update_time(bucket_entry.updated_at.as_deref()));
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&bucket_object,
|
||||
&bucket_entry,
|
||||
TableCatalogPutPrecondition::IfMatch(bucket_etag),
|
||||
_bucket_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -1927,12 +1873,7 @@ where
|
||||
let candidate = table_warehouse_index_entry(entry)?;
|
||||
validate_table_entry_version_and_id(entry)?;
|
||||
for existing in self
|
||||
.list_all_table_entries_with_limit(
|
||||
&candidate.table_bucket,
|
||||
None,
|
||||
locked_table_entry,
|
||||
TABLE_REGISTRATION_READ_CONCURRENCY,
|
||||
)
|
||||
.list_all_table_entries_with_limit(&candidate.table_bucket, None, locked_table_entry)
|
||||
.await?
|
||||
{
|
||||
if existing.state != TableCatalogEntryState::Active {
|
||||
@@ -1955,38 +1896,34 @@ where
|
||||
}
|
||||
}
|
||||
let state_object = self.paths.warehouse_index_state_path(&candidate.table_bucket);
|
||||
let index_objects = self
|
||||
for object in self
|
||||
.backend
|
||||
.list_objects(self.catalog_bucket(), &self.paths.warehouse_index_entries_prefix(&candidate.table_bucket))
|
||||
.await?;
|
||||
stream::iter(index_objects.into_iter().filter(|object| object != &state_object))
|
||||
.map(|object| {
|
||||
let candidate = &candidate;
|
||||
async move {
|
||||
let Some((existing, _)) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?;
|
||||
if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active {
|
||||
return Ok(());
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
&& self.warehouse_index_entry_has_active_owner(&existing).await?
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {}",
|
||||
candidate.warehouse_object_prefix
|
||||
)));
|
||||
}
|
||||
Ok::<_, TableCatalogStoreError>(())
|
||||
}
|
||||
})
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY)
|
||||
.try_for_each(|()| async { Ok(()) })
|
||||
.await
|
||||
.await?
|
||||
{
|
||||
if object == state_object {
|
||||
continue;
|
||||
}
|
||||
let Some((existing, _)) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object)
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?;
|
||||
if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
&& self.warehouse_index_entry_has_active_owner(&existing).await?
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {}",
|
||||
candidate.warehouse_object_prefix
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reserve_table_warehouse_index(
|
||||
@@ -2427,7 +2364,7 @@ where
|
||||
) -> TableCatalogStoreResult<Option<TableDataPlaneResource>> {
|
||||
let mut matched: Option<TableDataPlaneResource> = None;
|
||||
for table in self
|
||||
.list_all_table_entries_with_limit(table_bucket, Some(TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS), None, 1)
|
||||
.list_all_table_entries_with_limit(table_bucket, Some(TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS), None)
|
||||
.await?
|
||||
{
|
||||
if table.state != TableCatalogEntryState::Active {
|
||||
@@ -2519,7 +2456,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
let tables = self
|
||||
.list_all_table_entries_with_limit(table_bucket, max_catalog_objects, None, 1)
|
||||
.list_all_table_entries_with_limit(table_bucket, max_catalog_objects, None)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|table| table.state == TableCatalogEntryState::Active)
|
||||
@@ -2600,7 +2537,7 @@ where
|
||||
}
|
||||
|
||||
async fn list_all_table_entries(&self, table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
|
||||
self.list_all_table_entries_with_limit(table_bucket, None, None, 1).await
|
||||
self.list_all_table_entries_with_limit(table_bucket, None, None).await
|
||||
}
|
||||
|
||||
async fn list_table_entry_objects_for_data_plane_scan(
|
||||
@@ -2649,8 +2586,8 @@ where
|
||||
table_bucket: &str,
|
||||
max_catalog_objects: Option<usize>,
|
||||
locked_table_entry: Option<&str>,
|
||||
read_concurrency: usize,
|
||||
) -> TableCatalogStoreResult<Vec<TableEntry>> {
|
||||
let mut entries = Vec::new();
|
||||
let table_objects = match max_catalog_objects {
|
||||
Some(max_catalog_objects) => {
|
||||
self.list_table_entry_objects_for_data_plane_scan(table_bucket, max_catalog_objects)
|
||||
@@ -2664,24 +2601,19 @@ where
|
||||
.filter(|object| object.ends_with(TABLE_ENTRY_FILE))
|
||||
.collect(),
|
||||
};
|
||||
let mut entries = stream::iter(table_objects)
|
||||
.map(|object| async move {
|
||||
// Only the exact entry protected by the caller's write lock can bypass a read lock.
|
||||
let entry = if locked_table_entry == Some(object.as_str()) {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
};
|
||||
let Some((entry, _)) = entry else {
|
||||
return Ok(None);
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &entry)?;
|
||||
Ok::<_, TableCatalogStoreError>(Some(entry))
|
||||
})
|
||||
.buffered(read_concurrency)
|
||||
.try_filter_map(|entry| async move { Ok(entry) })
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
for object in table_objects {
|
||||
// Only the exact entry protected by the caller's write lock can bypass a read lock.
|
||||
let entry = if locked_table_entry == Some(object.as_str()) {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
};
|
||||
let Some((entry, _)) = entry else {
|
||||
continue;
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &entry)?;
|
||||
entries.push(entry);
|
||||
}
|
||||
entries.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table)));
|
||||
Ok(entries)
|
||||
}
|
||||
@@ -2840,7 +2772,8 @@ where
|
||||
"table registration requires a table publication fence".to_string(),
|
||||
));
|
||||
}
|
||||
self.check_registration_ownership(&entry, &table_path, publication).await?;
|
||||
self.ensure_table_warehouse_prefix_available(&entry, Some(&table_path))
|
||||
.await?;
|
||||
let reservation = self.reserve_table_warehouse_index(&entry, true, &table_path).await?;
|
||||
if !publication.holds_table_bucket(&entry.table_bucket)
|
||||
|| !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table)
|
||||
@@ -2851,23 +2784,20 @@ where
|
||||
"table registration publication fence was lost before catalog update".to_string(),
|
||||
));
|
||||
}
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let result = self
|
||||
.write_entry_unlocked_with_fence(
|
||||
.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&table_path,
|
||||
&entry,
|
||||
precondition,
|
||||
_table_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await;
|
||||
if result.is_err() {
|
||||
self.rollback_table_warehouse_index_reservation(&entry, reservation, "table entry write failed")
|
||||
.await;
|
||||
}
|
||||
result?;
|
||||
Ok(())
|
||||
result
|
||||
}
|
||||
|
||||
async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> {
|
||||
@@ -6328,15 +6258,27 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn disable_empty_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
|
||||
let mut disablement = self.begin_table_bucket_disablement(entry, None).await?;
|
||||
async fn disable_empty_table_bucket(&self, mut entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
|
||||
validate_table_bucket_entry(&entry)?;
|
||||
let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?;
|
||||
let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?;
|
||||
let object = self.paths.table_bucket_entry_path(&entry.table_bucket);
|
||||
let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?;
|
||||
if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? {
|
||||
if current.active_rename_id.is_some()
|
||||
|| !matches!(current.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict("table bucket has an operation in progress".to_string()));
|
||||
}
|
||||
entry = current;
|
||||
}
|
||||
// Namespace creation holds this same bucket lock. Retained table, view, and
|
||||
// maintenance records also prevent removing their data-plane protections.
|
||||
let page = self
|
||||
.backend
|
||||
.list_objects_page(
|
||||
self.catalog_bucket(),
|
||||
&self.paths.namespace_entries_prefix(&disablement.entry.table_bucket),
|
||||
&self.paths.namespace_entries_prefix(&entry.table_bucket),
|
||||
None,
|
||||
NonZeroUsize::MIN,
|
||||
)
|
||||
@@ -6346,7 +6288,16 @@ where
|
||||
"table bucket contains namespaces or retained catalog resources".to_string(),
|
||||
));
|
||||
}
|
||||
self.finish_table_bucket_disablement(&mut disablement).await
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
entry.updated_at = Some(next_table_catalog_update_time(entry.updated_at.as_deref()));
|
||||
self.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&object,
|
||||
&entry,
|
||||
TableCatalogPutPrecondition::Any,
|
||||
_guard.write_commit_guards(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> {
|
||||
@@ -6730,7 +6681,6 @@ where
|
||||
));
|
||||
}
|
||||
let _publication_completion = TableCommitPublicationCompletion::new(&publication);
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
self.recover_active_table_rename(table_bucket, &publication).await?;
|
||||
|
||||
{
|
||||
@@ -6870,13 +6820,12 @@ where
|
||||
));
|
||||
}
|
||||
None => {
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&index_object,
|
||||
&source_index,
|
||||
TableCatalogPutPrecondition::IfAbsent,
|
||||
_index_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
@@ -6897,24 +6846,22 @@ where
|
||||
validate_table_rename_intent_object(&self.paths, &intent_object, &intent)?;
|
||||
|
||||
// An orphan intent has no catalog effect; a published bucket fence without its intent cannot be recovered safely.
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&intent_object,
|
||||
&intent,
|
||||
TableCatalogPutPrecondition::IfAbsent,
|
||||
_intent_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
bucket_entry.active_rename_id = Some(rename_id);
|
||||
bucket_entry.updated_at = Some(now);
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.write_exact_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&bucket_object,
|
||||
&bucket_entry,
|
||||
TableCatalogPutPrecondition::IfMatch(bucket_etag),
|
||||
_bucket_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -7364,15 +7311,13 @@ where
|
||||
}
|
||||
|
||||
let cas_started = Instant::now();
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let cas_result = self
|
||||
.write_entry_unlocked_with_fence(
|
||||
.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&table_path,
|
||||
&next,
|
||||
TableCatalogPutPrecondition::IfMatch(current_etag),
|
||||
_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await;
|
||||
record_table_commit_cas_result(&request.operation, cas_started, &cas_result);
|
||||
@@ -7443,7 +7388,6 @@ where
|
||||
table.as_str()
|
||||
)));
|
||||
};
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
self.tombstone_table_warehouse_index_for_drop(&entry, false).await?;
|
||||
if !publication.holds_table_bucket(table_bucket)
|
||||
|| !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str())
|
||||
@@ -7454,11 +7398,7 @@ where
|
||||
"table drop publication fence was lost before catalog update".to_string(),
|
||||
));
|
||||
}
|
||||
if let Err(err) = self
|
||||
.backend
|
||||
.delete_object_unlocked_fenced(self.catalog_bucket(), &object, _guard.write_commit_guards(), &mutation_fence)
|
||||
.await
|
||||
{
|
||||
if let Err(err) = self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await {
|
||||
match self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await {
|
||||
Ok(None) => return Ok(()),
|
||||
Ok(Some((current, _))) if current == entry => {
|
||||
|
||||
@@ -1,298 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::object::{
|
||||
TABLE_REGISTRATION_READ_CONCURRENCY, table_warehouse_index_state_ready, validate_table_entry_object,
|
||||
validate_table_warehouse_index_entry_object,
|
||||
};
|
||||
use super::*;
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const REGISTRATION_CACHE_BYTES: u64 = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedOwner {
|
||||
etag: String,
|
||||
owner: Arc<TableWarehouseIndexEntry>,
|
||||
}
|
||||
|
||||
/// Runtime-local object projections. Every lookup requires a fresh committed inventory.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TableCatalogRegistrationCache {
|
||||
entries: Arc<OnceLock<moka::sync::Cache<String, CachedOwner>>>,
|
||||
capacity_bytes: u64,
|
||||
}
|
||||
|
||||
impl Default for TableCatalogRegistrationCache {
|
||||
fn default() -> Self {
|
||||
Self::with_capacity(REGISTRATION_CACHE_BYTES)
|
||||
}
|
||||
}
|
||||
|
||||
impl TableCatalogRegistrationCache {
|
||||
fn with_capacity(bytes: u64) -> Self {
|
||||
Self {
|
||||
entries: Arc::new(OnceLock::new()),
|
||||
capacity_bytes: bytes,
|
||||
}
|
||||
}
|
||||
|
||||
fn entries(&self) -> &moka::sync::Cache<String, CachedOwner> {
|
||||
self.entries.get_or_init(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(self.capacity_bytes)
|
||||
.weigher(|key: &String, value: &CachedOwner| {
|
||||
let owner = &value.owner;
|
||||
let bytes = [
|
||||
key.len(),
|
||||
value.etag.len(),
|
||||
owner.table_bucket.len(),
|
||||
owner.namespace.len(),
|
||||
owner.table.len(),
|
||||
owner.table_id.len(),
|
||||
owner.warehouse_object_prefix.len(),
|
||||
]
|
||||
.into_iter()
|
||||
.fold(256_usize, usize::saturating_add);
|
||||
u32::try_from(bytes).unwrap_or(u32::MAX)
|
||||
})
|
||||
.build()
|
||||
})
|
||||
}
|
||||
|
||||
fn get(&self, object: &TableCatalogCleanupObject) -> Option<Arc<TableWarehouseIndexEntry>> {
|
||||
let etag = object.etag.as_deref().filter(|etag| !etag.is_empty())?;
|
||||
self.entries
|
||||
.get()?
|
||||
.get(&object.key)
|
||||
.filter(|cached| cached.etag == etag)
|
||||
.map(|cached| cached.owner)
|
||||
}
|
||||
|
||||
fn insert(&self, object: String, etag: Option<String>, owner: Arc<TableWarehouseIndexEntry>) {
|
||||
if let Some(etag) = etag.filter(|etag| !etag.is_empty()) {
|
||||
self.entries().insert(object, CachedOwner { etag, owner });
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::table_catalog) fn clear(&self) {
|
||||
if let Some(entries) = self.entries.get() {
|
||||
entries.invalidate_all();
|
||||
entries.run_pending_tasks();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_inventory(objects: &mut [TableCatalogCleanupObject], prefix: &str) -> TableCatalogStoreResult<()> {
|
||||
objects.sort_unstable_by(|left, right| left.key.cmp(&right.key));
|
||||
if objects.iter().any(|object| {
|
||||
!object.key.starts_with(prefix)
|
||||
|| object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
}) || objects.windows(2).any(|pair| pair[0].key == pair[1].key)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"registration requires an unversioned, complete catalog inventory".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
|
||||
async fn registration_owner(
|
||||
&self,
|
||||
object: TableCatalogCleanupObject,
|
||||
table_record: bool,
|
||||
locked_table_entry: &str,
|
||||
cache: Option<&TableCatalogRegistrationCache>,
|
||||
) -> TableCatalogStoreResult<Arc<TableWarehouseIndexEntry>> {
|
||||
if let Some(owner) = cache.and_then(|cache| cache.get(&object)) {
|
||||
return Ok(owner);
|
||||
}
|
||||
let (owner, etag) = if table_record {
|
||||
let value = if object.key == locked_table_entry {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object.key).await?
|
||||
};
|
||||
let (entry, etag) = value.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("table record changed during registration inventory".to_string())
|
||||
})?;
|
||||
validate_table_entry_object(&self.paths, &object.key, &entry)?;
|
||||
(table_warehouse_index_entry(&entry)?, etag)
|
||||
} else {
|
||||
let (entry, etag) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("warehouse index changed during registration inventory".to_string())
|
||||
})?;
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object.key, &entry)?;
|
||||
(entry, etag)
|
||||
};
|
||||
let owner = Arc::new(owner);
|
||||
if let Some(cache) = cache {
|
||||
// A concurrent index repair may change its ETag after enumeration.
|
||||
// Associate the projection only with the object that GET validated.
|
||||
cache.insert(object.key, etag, Arc::clone(&owner));
|
||||
}
|
||||
Ok(owner)
|
||||
}
|
||||
|
||||
pub(super) async fn check_registration_ownership(
|
||||
&self,
|
||||
entry: &TableEntry,
|
||||
locked_table_entry: &str,
|
||||
publication: &(dyn TableCommitPublication + Sync),
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(TableCatalogStoreError::Unavailable("registration publication fence was lost".to_string()));
|
||||
}
|
||||
validate_table_entry_version_and_id(entry)?;
|
||||
let candidate = table_warehouse_index_entry(entry)?;
|
||||
let namespace_prefix = self.paths.namespace_entries_prefix(&entry.table_bucket);
|
||||
let index_prefix = self.paths.warehouse_index_entries_prefix(&entry.table_bucket);
|
||||
// The backend must preserve system-bucket committed-read semantics as
|
||||
// well as completeness; ordinary S3 LIST is insufficient for cache hits.
|
||||
let mut tables = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &namespace_prefix)
|
||||
.await?;
|
||||
let mut indexes = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &index_prefix)
|
||||
.await?;
|
||||
validate_inventory(&mut tables, &namespace_prefix)?;
|
||||
validate_inventory(&mut indexes, &index_prefix)?;
|
||||
let cache = self.backend.registration_cache();
|
||||
let mut owners = BTreeMap::new();
|
||||
let mut table_ids = BTreeSet::new();
|
||||
let mut table_reads = stream::iter(tables.into_iter().filter(|object| object.key.ends_with(TABLE_ENTRY_FILE)))
|
||||
.map(|object| self.registration_owner(object, true, locked_table_entry, cache.as_ref()))
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY);
|
||||
while let Some(owner) = table_reads.try_next().await? {
|
||||
if owner.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if !table_ids.insert(owner.table_id.clone())
|
||||
|| (owner.table_id == candidate.table_id
|
||||
&& (owner.namespace != candidate.namespace || owner.table != candidate.table))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table id is already registered in this table bucket".to_string(),
|
||||
));
|
||||
}
|
||||
if owner.table_id != candidate.table_id
|
||||
&& warehouse_object_prefixes_overlap(&owner.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table warehouse location overlaps an active table".to_string(),
|
||||
));
|
||||
}
|
||||
owners.insert((owner.namespace.clone(), owner.table.clone()), owner);
|
||||
}
|
||||
let mut prefixes = owners
|
||||
.values()
|
||||
.map(|owner| owner.warehouse_object_prefix.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
prefixes.sort_unstable();
|
||||
if prefixes
|
||||
.windows(2)
|
||||
.any(|pair| warehouse_object_prefixes_overlap(pair[0], pair[1]))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict("active table warehouse locations overlap".to_string()));
|
||||
}
|
||||
let state_path = self.paths.warehouse_index_state_path(&entry.table_bucket);
|
||||
let has_state = indexes.iter().any(|object| object.key == state_path);
|
||||
let mut index_reads = stream::iter(indexes.into_iter().filter(|object| object.key != state_path))
|
||||
.map(|object| self.registration_owner(object, false, locked_table_entry, cache.as_ref()))
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY);
|
||||
while let Some(index) = index_reads.try_next().await? {
|
||||
if index.table_id == candidate.table_id || index.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&index.warehouse_object_prefix, &candidate.warehouse_object_prefix) {
|
||||
parse_namespace_for_store(&index.namespace)?;
|
||||
parse_table_for_store(&index.table)?;
|
||||
if owners
|
||||
.get(&(index.namespace.clone(), index.table.clone()))
|
||||
.is_some_and(|owner| {
|
||||
owner.table_id == index.table_id && owner.warehouse_object_prefix == index.warehouse_object_prefix
|
||||
})
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table warehouse location overlaps an active table".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_state {
|
||||
let (state, _) = self
|
||||
.read_entry::<TableWarehouseIndexStateEntry>(self.catalog_bucket(), &state_path)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("warehouse index state changed during registration inventory".to_string())
|
||||
})?;
|
||||
table_warehouse_index_state_ready(&state, &entry.table_bucket)?;
|
||||
}
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(TableCatalogStoreError::Unavailable("registration publication fence was lost".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::registration_cache_tests::{cache_fixture, registration_entry};
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_capacity_and_oversized_owners_preserve_checks() {
|
||||
let (backend, _, namespace) = cache_fixture().await;
|
||||
let cache = TableCatalogRegistrationCache::with_capacity(1024);
|
||||
let backend = crate::table_catalog::test_support::TestCatalogObjectBackend {
|
||||
registration_cache: cache.clone(),
|
||||
..backend
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let mut oversized = registration_entry(&namespace, "oversized");
|
||||
oversized.warehouse_location = format!("s3://analytics/{}", "directory".repeat(512));
|
||||
store.create_table(oversized.clone()).await.unwrap();
|
||||
for number in 0..12 {
|
||||
store
|
||||
.create_table(registration_entry(&namespace, &format!("table_{number}")))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
cache.entries().run_pending_tasks();
|
||||
assert!(cache.entries().weighted_size() <= 1024);
|
||||
backend.reset_call_counts().await;
|
||||
let mut candidate = registration_entry(&namespace, "conflicting");
|
||||
candidate.warehouse_location = format!("{}/child", oversized.warehouse_location);
|
||||
assert!(matches!(store.create_table(candidate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
assert!(
|
||||
backend.read_call_count().await > 0,
|
||||
"eviction must cause reads instead of accepting an absent owner"
|
||||
);
|
||||
cache.entries().run_pending_tasks();
|
||||
assert!(cache.entries().weighted_size() <= 1024);
|
||||
}
|
||||
}
|
||||
@@ -1,376 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::{RecordingLeaseClient, TestCatalogObjectBackend};
|
||||
use crate::table_catalog::tests::{test_bucket_entry, test_namespace_entry, test_table_entry};
|
||||
use rustfs_lock::LockClient;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
pub(super) fn registration_entry(namespace: &Namespace, name: &str) -> TableEntry {
|
||||
let table = IdentifierSegment::parse(name).expect("valid test table name");
|
||||
let mut entry = test_table_entry(
|
||||
"analytics",
|
||||
namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("id-{name}");
|
||||
entry.warehouse_location = format!("s3://analytics/tables/id-{name}");
|
||||
entry
|
||||
}
|
||||
|
||||
pub(super) async fn cache_fixture() -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, Namespace) {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store.create_table(registration_entry(&namespace, "orders")).await.unwrap();
|
||||
store.create_table(registration_entry(&namespace, "warmup")).await.unwrap();
|
||||
(backend, store, namespace)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_binds_projection_to_the_get_etag() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
backend.registration_cache.clear();
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let key = (RUSTFS_META_BUCKET.to_string(), path.clone());
|
||||
let original = backend.state.lock().await.objects.get(&key).unwrap().clone();
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
let task_store = store.clone();
|
||||
let entry = registration_entry(&namespace, "unrelated");
|
||||
let task = tokio::spawn(async move { task_store.create_table(entry).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/changed".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
pause.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
backend.state.lock().await.objects.insert(key, original);
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/changed/child".to_string();
|
||||
store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("a GET projection cannot inherit an earlier inventory ETag");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_observes_changed_and_removed_objects() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/relocated".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/relocated/child".to_string();
|
||||
assert!(matches!(
|
||||
store.create_table(candidate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
backend.delete_object(RUSTFS_META_BUCKET, &path).await.unwrap();
|
||||
candidate.table_id = changed.table_id;
|
||||
store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("removed records cannot remain active through the cache");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_revalidates_missing_etags_and_corrupt_changed_indexes() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.etagless_objects
|
||||
.insert((RUSTFS_META_BUCKET.to_string(), path.clone()));
|
||||
backend.fail_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
backend.state.lock().await.etagless_objects.clear();
|
||||
let index = store.paths.warehouse_index_entry_path("analytics", "tables/id-orders/");
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &index, b"invalid json".to_vec())
|
||||
.await;
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_rejects_incomplete_and_versioned_inventories() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let namespace_prefix = store.paths.namespace_entries_prefix("analytics");
|
||||
let index_prefix = store.paths.warehouse_index_entries_prefix("analytics");
|
||||
for prefix in [&namespace_prefix, &index_prefix] {
|
||||
backend.state.lock().await.failed_inventory_prefixes.insert(prefix.clone());
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
backend.state.lock().await.failed_inventory_prefixes.clear();
|
||||
}
|
||||
for (version_id, delete_marker) in [(Some(Uuid::new_v4().to_string()), false), (None, true)] {
|
||||
backend.state.lock().await.inventory_overrides.insert(
|
||||
namespace_prefix.clone(),
|
||||
vec![TableCatalogCleanupObject {
|
||||
key: format!("{namespace_prefix}unexpected"),
|
||||
version_id,
|
||||
etag: Some("etag".to_string()),
|
||||
delete_marker,
|
||||
}],
|
||||
);
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
}
|
||||
backend.state.lock().await.inventory_overrides.clear();
|
||||
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_none());
|
||||
let mut duplicate = registration_entry(&namespace, "candidate");
|
||||
duplicate.table_id = "id-orders".to_string();
|
||||
assert!(matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_cancellation_does_not_publish_a_partial_catalog() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
backend.registration_cache.clear();
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("warmup").unwrap());
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
let task_store = store.clone();
|
||||
let candidate = registration_entry(&namespace, "cancelled");
|
||||
let task = tokio::spawn(async move { task_store.create_table(candidate).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
pause.release();
|
||||
assert!(store.load_table("analytics", "sales", "cancelled").await.unwrap().is_none());
|
||||
let mut changed = registration_entry(&namespace, "warmup");
|
||||
changed.warehouse_location = "s3://analytics/changed".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
let mut candidate = registration_entry(&namespace, "after_cancel");
|
||||
candidate.warehouse_location = "s3://analytics/changed/child".to_string();
|
||||
assert!(matches!(store.create_table(candidate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_isolated_runtimes_do_not_share_matching_etags() {
|
||||
let (first, first_store, namespace) = cache_fixture().await;
|
||||
let (second, second_store, _) = cache_fixture().await;
|
||||
let path = first_store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/second-only".to_string();
|
||||
let first_etag = first
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.objects
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), path.clone()))
|
||||
.unwrap()
|
||||
.etag
|
||||
.clone();
|
||||
second.registration_cache.clear();
|
||||
{
|
||||
let mut state = second.state.lock().await;
|
||||
let object = state.objects.get_mut(&(RUSTFS_META_BUCKET.to_string(), path)).unwrap();
|
||||
object.data = serde_json::to_vec(&changed).unwrap();
|
||||
object.etag = first_etag;
|
||||
}
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/second-only/child".to_string();
|
||||
assert!(matches!(
|
||||
second_store.create_table(candidate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
first_store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("the first runtime has no such owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_preserves_identity_after_lost_response_and_failed_rollback() {
|
||||
for lost_response in [false, true] {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let entry = registration_entry(&namespace, "ambiguous");
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse(&entry.table).unwrap());
|
||||
let index = store.paths.warehouse_index_entry_path("analytics", "tables/id-ambiguous/");
|
||||
if lost_response {
|
||||
backend.fail_after_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
} else {
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
}
|
||||
backend.fail_delete_attempt(RUSTFS_META_BUCKET, &index, 1).await;
|
||||
assert!(store.create_table(entry.clone()).await.is_err());
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.table_id = entry.table_id;
|
||||
let result = store.create_table(candidate).await;
|
||||
if lost_response {
|
||||
assert!(matches!(result, Err(TableCatalogStoreError::Conflict(_))));
|
||||
} else {
|
||||
result.expect("failed primary writes cannot reserve identity through cached results");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn leased_registration_publication<'a>(
|
||||
backend: &'a TestCatalogObjectBackend,
|
||||
lock: &rustfs_lock::NamespaceLock,
|
||||
owner: &str,
|
||||
) -> (
|
||||
TableCommitLockPublication<'a, TestCatalogObjectBackend>,
|
||||
Arc<rustfs_lock::distributed_lock::LockLostSignal>,
|
||||
) {
|
||||
let guard = lock
|
||||
.lock_guard(
|
||||
rustfs_lock::ObjectKey::new("analytics", default_table_bucket_publication_lock_path()),
|
||||
owner,
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(3),
|
||||
)
|
||||
.await
|
||||
.expect("publication lease acquisition succeeds")
|
||||
.expect("publication lease is acquired");
|
||||
let signal = guard.lock_lost_signal().expect("distributed publication has a loss signal");
|
||||
let publication = TableCommitLockPublication::new(backend);
|
||||
{
|
||||
let mut state = publication.state.lock();
|
||||
state.table_bucket = Some("analytics".to_string());
|
||||
state.guards.push(TableCatalogLockGuard::namespace(guard));
|
||||
}
|
||||
(publication, signal)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn registration_cache_rejects_delayed_table_write_after_publication_lease_loss() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let stalled_namespace = Namespace::parse("stalled").unwrap();
|
||||
let active_namespace = Namespace::parse("active").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
|
||||
for namespace in [&stalled_namespace, &active_namespace] {
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let initial = registration_entry(&active_namespace, "initial");
|
||||
store.create_table(initial.clone()).await.unwrap();
|
||||
let client = Arc::new(RecordingLeaseClient {
|
||||
inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)))),
|
||||
acquired: Mutex::new(None),
|
||||
});
|
||||
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"registration-publication-loss".to_string(),
|
||||
vec![client.clone() as Arc<dyn LockClient>],
|
||||
1,
|
||||
);
|
||||
let (old_publication, old_signal) = leased_registration_publication(&backend, &lock, "old-writer").await;
|
||||
let old_lease = client.acquired.lock().unwrap().clone().unwrap();
|
||||
let delayed = registration_entry(&stalled_namespace, "delayed");
|
||||
let delayed_path =
|
||||
store
|
||||
.paths
|
||||
.table_entry_path("analytics", &stalled_namespace, &IdentifierSegment::parse(&delayed.table).unwrap());
|
||||
let pause = backend.pause_put_attempt(RUSTFS_META_BUCKET, &delayed_path, 1).await;
|
||||
let replacement = registration_entry(&active_namespace, "replacement");
|
||||
let delayed_write = store.register_table_with_publication(delayed.clone(), &old_publication);
|
||||
let takeover = async {
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.expect("old writer reaches the table-entry commit");
|
||||
assert!(client.force_release(&old_lease).await.unwrap());
|
||||
tokio::time::timeout(Duration::from_secs(10), old_signal.notified())
|
||||
.await
|
||||
.expect("heartbeat observes publication lease loss");
|
||||
assert!(!old_publication.holds_table_bucket("analytics"));
|
||||
|
||||
let (new_publication, _) = leased_registration_publication(&backend, &lock, "new-writer").await;
|
||||
backend.reset_call_counts().await;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
store.register_table_with_publication(replacement.clone(), &new_publication),
|
||||
)
|
||||
.await
|
||||
.expect("replacement writer completes while the old put is paused")
|
||||
.expect("replacement writer publishes its table after a fresh inventory");
|
||||
assert!(backend.list_call_count().await > 0, "takeover inventories the catalog");
|
||||
pause.release();
|
||||
};
|
||||
let (delayed_result, ()) = tokio::time::timeout(Duration::from_secs(25), async { tokio::join!(delayed_write, takeover) })
|
||||
.await
|
||||
.expect("both writers finish after the paused put is resumed");
|
||||
assert!(
|
||||
matches!(delayed_result, Err(TableCatalogStoreError::Unavailable(_))),
|
||||
"the delayed storage commit must reject the lost publication fence: {delayed_result:?}"
|
||||
);
|
||||
assert_eq!(backend.put_attempt_count(RUSTFS_META_BUCKET, &delayed_path).await, 1);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(RUSTFS_META_BUCKET, &delayed_path)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for entry in [&initial, &replacement] {
|
||||
assert_eq!(
|
||||
store.load_table("analytics", &entry.namespace, &entry.table).await.unwrap(),
|
||||
Some(entry.clone())
|
||||
);
|
||||
}
|
||||
|
||||
let subsequent = registration_entry(&active_namespace, "subsequent");
|
||||
store.create_table(subsequent.clone()).await.unwrap();
|
||||
for entry in [&initial, &replacement, &subsequent] {
|
||||
let mut duplicate = registration_entry(&active_namespace, "duplicate");
|
||||
duplicate.table_id = entry.table_id.clone();
|
||||
assert!(matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
store
|
||||
.create_table(delayed)
|
||||
.await
|
||||
.expect("the rejected writer left no active table");
|
||||
}
|
||||
@@ -411,7 +411,6 @@ pub(crate) struct TestCatalogObjectBackend {
|
||||
pub(crate) state: Arc<tokio::sync::Mutex<TestCatalogObjectState>>,
|
||||
pub(crate) locks: TestCatalogObjectLocks,
|
||||
pub(crate) strong_runtime: Option<StrongTableCatalogRuntime>,
|
||||
pub(crate) registration_cache: TableCatalogRegistrationCache,
|
||||
// One-shot, path-keyed injection knobs from the admin handler tests'
|
||||
// former TestTableCatalogObjectBackend (backlog#1837 PR2): each fires
|
||||
// once for the named object and clears itself, mirroring the original
|
||||
@@ -544,11 +543,8 @@ pub(crate) struct TestCatalogObjectState {
|
||||
pub(crate) write_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_calls: usize,
|
||||
pub(crate) read_bytes: usize,
|
||||
pub(crate) metadata_calls: usize,
|
||||
pub(crate) list_calls: usize,
|
||||
pub(crate) failed_inventory_prefixes: BTreeSet<String>,
|
||||
pub(crate) inventory_overrides: BTreeMap<String, Vec<TableCatalogCleanupObject>>,
|
||||
pub(crate) next_etag: u64,
|
||||
}
|
||||
|
||||
@@ -612,7 +608,6 @@ impl TestCatalogObjectBackend {
|
||||
pub(crate) async fn reset_call_counts(&self) {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls = 0;
|
||||
state.read_bytes = 0;
|
||||
state.metadata_calls = 0;
|
||||
state.list_calls = 0;
|
||||
}
|
||||
@@ -960,10 +955,6 @@ impl TestCatalogObjectState {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
Some(self.registration_cache.clone())
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
self.strong_runtime.clone()
|
||||
}
|
||||
@@ -1027,7 +1018,6 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
});
|
||||
state.read_bytes += result.as_ref().map_or(0, |object| object.data.len());
|
||||
let pause = state
|
||||
.pause_read_attempts
|
||||
.get_mut(&key)
|
||||
@@ -1152,17 +1142,6 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
_expected_incarnation_id: Uuid,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object_if_match_fenced(bucket, object, expected_etag, fence).await
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
@@ -1174,38 +1153,6 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
if state.failed_inventory_prefixes.contains(prefix) {
|
||||
return Err(TableCatalogStoreError::Unavailable(
|
||||
"injected authoritative inventory failure".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(objects) = state.inventory_overrides.get(prefix) {
|
||||
return Ok(objects.clone());
|
||||
}
|
||||
Ok(state
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|((entry_bucket, object), _)| entry_bucket == bucket && object.starts_with(prefix))
|
||||
.map(|(key, record)| TableCatalogCleanupObject {
|
||||
key: key.1.clone(),
|
||||
version_id: None,
|
||||
etag: (!state.etagless_objects.contains(key)).then(|| record.etag.clone()),
|
||||
delete_marker: false,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, _bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<TableCatalogLockGuard> {
|
||||
self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string()));
|
||||
{
|
||||
|
||||
@@ -7145,228 +7145,6 @@ async fn table_data_plane_resource_does_not_match_sibling_prefix() {
|
||||
assert_eq!(backend.list_call_count().await, 1);
|
||||
}
|
||||
|
||||
async fn cached_registration_fixture() -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, Namespace)
|
||||
{
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let namespace = Namespace::parse("sales").expect("test namespace is valid");
|
||||
store
|
||||
.put_table_bucket(test_bucket_entry("analytics"))
|
||||
.await
|
||||
.expect("bucket is created");
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", &namespace))
|
||||
.await
|
||||
.expect("namespace is created");
|
||||
(backend, store, namespace)
|
||||
}
|
||||
|
||||
fn cached_registration_entry(namespace: &Namespace, name: &str) -> TableEntry {
|
||||
let table = IdentifierSegment::parse(name).expect("test table name is valid");
|
||||
let mut entry = test_table_entry(
|
||||
"analytics",
|
||||
namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("id-{name}");
|
||||
entry.warehouse_location = format!("s3://analytics/tables/id-{name}");
|
||||
entry
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_bounds_reads_across_store_instances() {
|
||||
let (backend, _, namespace) = cached_registration_fixture().await;
|
||||
for number in 0..256 {
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, &format!("table_{number:04}")))
|
||||
.await
|
||||
.expect("each new store instance shares validated registration projections");
|
||||
if number >= 32 {
|
||||
let state = backend.state.lock().await;
|
||||
assert_eq!(state.list_calls, 2, "each registration inventories both catalog ranges");
|
||||
assert!(
|
||||
state.read_calls < 24,
|
||||
"registration used {} object reads at table {number}",
|
||||
state.read_calls
|
||||
);
|
||||
assert!(
|
||||
state.read_bytes < 256 * 1024,
|
||||
"registration read {} bytes at table {number}",
|
||||
state.read_bytes
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_preserves_identity_and_overlap_checks() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
let mut existing = cached_registration_entry(&namespace, "orders");
|
||||
existing.warehouse_location = "s3://analytics/shared/orders".to_string();
|
||||
store.create_table(existing.clone()).await.expect("initial table is created");
|
||||
backend
|
||||
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path("analytics", "shared/orders/"))
|
||||
.await
|
||||
.expect("simulate a missing materialized warehouse index");
|
||||
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = existing.table_id.clone();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
for location in [
|
||||
"s3://analytics/shared",
|
||||
"s3://analytics/shared/orders",
|
||||
"s3://analytics/shared/orders/child",
|
||||
] {
|
||||
let mut conflicting = cached_registration_entry(&namespace, "conflicting");
|
||||
conflicting.warehouse_location = location.to_string();
|
||||
assert_matches!(store.create_table(conflicting).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
let mut adjacent = cached_registration_entry(&namespace, "adjacent");
|
||||
adjacent.warehouse_location = "s3://analytics/shared/orders2".to_string();
|
||||
store
|
||||
.create_table(adjacent)
|
||||
.await
|
||||
.expect("a different directory remains available");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_rechecks_after_failed_write_and_drop() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "orders"))
|
||||
.await
|
||||
.expect("initial table is created");
|
||||
let returns = IdentifierSegment::parse("returns").expect("test name is valid");
|
||||
let table_path = store.paths.table_entry_path("analytics", &namespace, &returns);
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
|
||||
assert!(
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.load_table("analytics", "sales", "returns")
|
||||
.await
|
||||
.expect("table lookup succeeds")
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.expect("retry reads the authoritative records");
|
||||
store
|
||||
.drop_table("analytics", "sales", "orders")
|
||||
.await
|
||||
.expect("drop removes the active owner");
|
||||
let mut replacement = cached_registration_entry(&namespace, "replacement");
|
||||
replacement.table_id = "id-orders".to_string();
|
||||
replacement.warehouse_location = "s3://analytics/tables/id-orders".to_string();
|
||||
store
|
||||
.create_table(replacement)
|
||||
.await
|
||||
.expect("a dropped owner no longer reserves its identity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_observes_pointer_commits_and_rename() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
let entry = cached_registration_entry(&namespace, "orders");
|
||||
store.create_table(entry.clone()).await.expect("initial table is created");
|
||||
let metadata_directory = entry
|
||||
.metadata_location
|
||||
.rsplit_once('/')
|
||||
.expect("metadata has a parent directory")
|
||||
.0;
|
||||
let next_metadata = format!("{metadata_directory}/00002.metadata.json");
|
||||
backend
|
||||
.seed_object(
|
||||
"analytics",
|
||||
&next_metadata,
|
||||
serde_json::to_vec(&serde_json::json!({"location": entry.warehouse_location})).unwrap(),
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.commit_table(TableCommitRequest {
|
||||
table_bucket: "analytics".to_string(),
|
||||
namespace: "sales".to_string(),
|
||||
table: "orders".to_string(),
|
||||
commit_id: "pointer-only".to_string(),
|
||||
idempotency_key: None,
|
||||
operation: "append".to_string(),
|
||||
expected_version_token: entry.version_token,
|
||||
expected_metadata_location: entry.metadata_location,
|
||||
new_metadata_location: next_metadata,
|
||||
requirements: Vec::new(),
|
||||
writer: None,
|
||||
})
|
||||
.await
|
||||
.expect("metadata pointer commit succeeds");
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
backend.list_call_count().await,
|
||||
2,
|
||||
"pointer changes are checked through fresh inventories"
|
||||
);
|
||||
|
||||
store
|
||||
.rename_table("analytics", "sales", "orders", "sales", "renamed")
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "shipments"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
backend.list_call_count().await > 0,
|
||||
"rename is visible in the next authoritative inventory"
|
||||
);
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = "id-orders".to_string();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_observes_an_independent_writer() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "orders"))
|
||||
.await
|
||||
.unwrap();
|
||||
let scan_store = ObjectTableCatalogStore::new(TestCatalogObjectBackend {
|
||||
registration_cache: TableCatalogRegistrationCache::default(),
|
||||
..backend.clone()
|
||||
});
|
||||
scan_store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "shipments"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
backend.list_call_count().await > 0,
|
||||
"an independent writer is visible in the next authoritative inventory"
|
||||
);
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = "id-returns".to_string();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_scans_warehouse_prefixes_once() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
@@ -7395,217 +7173,10 @@ async fn object_catalog_registration_scans_warehouse_prefixes_once() {
|
||||
reads.push(backend.read_call_count().await);
|
||||
assert_eq!(backend.list_call_count().await, 2, "one table scan and one warehouse index scan");
|
||||
}
|
||||
assert_eq!(reads[2], reads[1], "unchanged validated objects should not require another GET");
|
||||
assert_eq!(reads[2] - reads[1], 2, "each existing table and its index should be read once");
|
||||
assert_eq!(store.list_tables(bucket, &namespace.public_name()).await.unwrap().len(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_bounds_concurrent_catalog_reads() {
|
||||
const MAX_CONCURRENT_READS: usize = 8;
|
||||
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut table_objects = Vec::new();
|
||||
let mut index_objects = Vec::new();
|
||||
for number in 0..MAX_CONCURRENT_READS + 2 {
|
||||
let name = format!("orders_{number:02}");
|
||||
let table = IdentifierSegment::parse(&name).unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("table-{name}");
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/{name}");
|
||||
store.create_table(entry).await.unwrap();
|
||||
table_objects.push(store.paths.table_entry_path(bucket, &namespace, &table));
|
||||
index_objects.push(store.paths.warehouse_index_entry_path(bucket, &format!("tables/{name}/")));
|
||||
}
|
||||
|
||||
backend.registration_cache.clear();
|
||||
let mut scans = Vec::new();
|
||||
for mut objects in [table_objects, index_objects] {
|
||||
objects.sort();
|
||||
let mut scan = Vec::new();
|
||||
for object in objects {
|
||||
let previous_reads = backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.read_attempts
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), object.clone()))
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &object).await;
|
||||
scan.push((object, pause, previous_reads));
|
||||
}
|
||||
scans.push(scan);
|
||||
}
|
||||
|
||||
let table = IdentifierSegment::parse("new_orders").unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = "new-table-id".to_string();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/new_orders");
|
||||
let registering_store = store.clone();
|
||||
let registration = tokio::spawn(async move { registering_store.create_table(entry).await });
|
||||
|
||||
for scan in scans {
|
||||
for (_, pause, _) in scan.iter().take(MAX_CONCURRENT_READS) {
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
|
||||
.await
|
||||
.expect("independent catalog reads should start concurrently");
|
||||
}
|
||||
let state = backend.state.lock().await;
|
||||
let started = scan
|
||||
.iter()
|
||||
.filter(|(object, _, previous_reads)| {
|
||||
state
|
||||
.read_attempts
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), object.clone()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
> *previous_reads
|
||||
})
|
||||
.count();
|
||||
assert_eq!(started, MAX_CONCURRENT_READS, "catalog reads must respect the concurrency limit");
|
||||
drop(state);
|
||||
for (_, pause, _) in scan {
|
||||
pause.release();
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, registration)
|
||||
.await
|
||||
.expect("registration should finish after its reads are released")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_propagates_catalog_scan_read_errors() {
|
||||
for fail_index_read in [false, true] {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
let existing = IdentifierSegment::parse("orders").unwrap();
|
||||
seed_table_for_metadata_maintenance(
|
||||
&store,
|
||||
bucket,
|
||||
&namespace,
|
||||
&existing,
|
||||
default_table_metadata_file_path(&namespace, &existing, "00001.metadata.json"),
|
||||
)
|
||||
.await;
|
||||
let failed_object = if fail_index_read {
|
||||
store.paths.warehouse_index_entry_path(bucket, "tables/table-id/")
|
||||
} else {
|
||||
store.paths.table_entry_path(bucket, &namespace, &existing)
|
||||
};
|
||||
backend.fail_next_read(RUSTFS_META_BUCKET, &failed_object).await;
|
||||
|
||||
let table = IdentifierSegment::parse("returns").unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = "new-table-id".to_string();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/new-table-id");
|
||||
assert_matches!(
|
||||
store.create_table(entry.clone()).await,
|
||||
Err(TableCatalogStoreError::Internal(message)) if message.contains(&failed_object)
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&store.paths.warehouse_index_entry_path(bucket, "tables/new-table-id/")
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.create_table(entry)
|
||||
.await
|
||||
.expect("a failed scan must release its read locks for retry");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_reuses_locked_deleted_entry() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
let table = IdentifierSegment::parse("orders").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
backend
|
||||
.seed_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&store.paths.table_entry_path(bucket, &namespace, &table),
|
||||
serde_json::to_vec(&entry).unwrap(),
|
||||
)
|
||||
.await;
|
||||
entry.state = TableCatalogEntryState::Active;
|
||||
entry.table_id = "replacement-table-id".to_string();
|
||||
store.create_table(entry.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(entry)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_backings_reject_overlapping_registered_warehouse_prefixes() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
@@ -20955,9 +20526,9 @@ async fn object_table_catalog_store_reuses_old_prefix_after_failed_relocation_in
|
||||
let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current_metadata);
|
||||
next_entry.table_id = "next-table-id".to_string();
|
||||
next_entry.warehouse_location = format!("s3://{bucket}/tables/table-id");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), store.create_table(next_entry))
|
||||
store
|
||||
.create_table(next_entry)
|
||||
.await
|
||||
.expect("stale warehouse prefix reuse completes")
|
||||
.expect("stale old warehouse index should not block prefix reuse");
|
||||
|
||||
let (index, _) = store
|
||||
|
||||
@@ -22,7 +22,7 @@ class RegistryTests(unittest.TestCase):
|
||||
def test_all_workflow_responsibilities_and_suite_exits_are_wired(self):
|
||||
self.assertTrue(registry.check_wiring(self.registry))
|
||||
self.assertEqual(len(self.registry["suites"]), 12)
|
||||
self.assertEqual(sum(item["origin"] == "baseline" for item in self.registry["workflows"]), 61)
|
||||
self.assertEqual(sum(item["origin"] == "baseline" for item in self.registry["workflows"]), 60)
|
||||
self.assertEqual(len(self.registry["workflows"]), len(list((registry.ROOT / ".github/workflows").glob("*.yml"))))
|
||||
|
||||
def test_manual_and_periodic_plan_choose_identical_responsibilities(self):
|
||||
|
||||
Reference in New Issue
Block a user