mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-02 09:04:14 +00:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 91f2d53500 | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 | |||
| 3c0829049d | |||
| 890a463218 | |||
| 25f45f6309 | |||
| e3700614a1 | |||
| d4ab7704b3 | |||
| 422e724051 | |||
| e0bf54197d | |||
| b4af43c884 | |||
| 1cb4c0ebe2 | |||
| e5563e6004 | |||
| 0a72b29a34 | |||
| 51275cb7d8 | |||
| 3bf270b49f | |||
| d8d064bd18 | |||
| e5653ca055 | |||
| 6413a11889 | |||
| 9e4ea9fdd1 | |||
| 1a841fb7c7 | |||
| b8ee0301e9 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
|
||||
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
|
||||
+17
-5
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -417,6 +417,12 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-heal) & (binary(mrf_partial_write_test) | (binary(mrf_pipeline_test) & test(/^journal_replay_survives_/)) | test(=admin_selector_survives_sigkill_with_original_scope_and_receipts))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Bound the whole recovery fixture even if a synchronous wait prevents its
|
||||
# per-case async deadline from firing.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-heal) & binary(heal_integration_test) & test(=serial_tests::test_admin_bucket_directory_recovery_from_task_entry)'
|
||||
slow-timeout = { period = "60s", terminate-after = 5, grace-period = "10s" }
|
||||
|
||||
# Native scanner backlog fixtures include 24-set and pool-expansion cases.
|
||||
# Reserve capacity across test processes while preserving each scenario's fixed
|
||||
# budget, assertions, and internal concurrency.
|
||||
@@ -548,7 +554,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -970,10 +976,16 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
@@ -296,6 +296,32 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
|
||||
+311
-5
@@ -814,13 +814,18 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -869,6 +874,221 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1028,6 +1248,10 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1068,6 +1292,45 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1084,16 +1347,59 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status)
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
+36
-34
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
@@ -193,13 +193,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -233,11 +233,11 @@ jobs:
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ test-and-lint, classify-changes ]
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
@@ -293,14 +293,14 @@ jobs:
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
@@ -321,13 +321,13 @@ jobs:
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
@@ -368,7 +368,7 @@ jobs:
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/test-and-lint/core-test-listing.json \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
@@ -449,7 +449,7 @@ jobs:
|
||||
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -499,11 +499,13 @@ jobs:
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -540,12 +542,12 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
@@ -555,14 +557,14 @@ jobs:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -631,7 +633,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -783,7 +785,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -978,7 +980,7 @@ jobs:
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
@@ -1389,7 +1391,7 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
@@ -1425,7 +1427,7 @@ jobs:
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
|
||||
@@ -329,13 +329,75 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: build-check
|
||||
needs: [ build-check, release-certification ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -443,6 +505,14 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -467,7 +537,9 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -493,6 +565,114 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -30,9 +30,102 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
+210
-14
@@ -225,10 +225,57 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: resolve
|
||||
needs: [ resolve, release-certification ]
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -494,6 +541,55 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -531,10 +627,14 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -549,18 +649,6 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -614,6 +702,114 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
candidate = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = candidate["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + candidate["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
Generated
+51
-43
@@ -1321,9 +1321,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "1.1.0"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4"
|
||||
checksum = "9543fc722ed4865ab29fb66bd7ed0bef7b029d265ba8c587144511e30006d431"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
@@ -4198,9 +4198,9 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.42"
|
||||
version = "0.8.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
|
||||
checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
@@ -5277,9 +5277,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.30.1"
|
||||
version = "0.30.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "502dc37a7c218b9b1d3a0ccfb904da81d10d6dda0e899c722c503092d55576ea"
|
||||
checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5312,15 +5312,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.30.1"
|
||||
version = "0.30.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ea17d04c231b90b1f68a91f6f80fb0762c5dea07c611dce9ae157b56a50ff4c"
|
||||
checksum = "d0faae1a9e1fd8ce5debaaaf452dcbe833fc8a8a4f3779d2ec8828b116c3c16a"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.30.1"
|
||||
version = "0.30.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a76e0af911930fc97abc766b393b48cdca77fea9f5f6b388cf2123a6d9c30deb"
|
||||
checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5899,9 +5899,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.106"
|
||||
version = "0.3.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
|
||||
checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -6094,9 +6094,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libbz2-rs-sys"
|
||||
version = "0.2.5"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
|
||||
checksum = "32f454fd6831a29c0e827ed075bca2a15be201bd57f4168f0fd1041124eab17e"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
@@ -8305,9 +8305,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark"
|
||||
version = "0.13.5"
|
||||
version = "0.13.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb"
|
||||
checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"memchr",
|
||||
@@ -8823,9 +8823,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.8.0"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
|
||||
checksum = "db7f3d75b5a3301746bff39a767bf4c5c42506b08513527308b3875bfb64a031"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -12217,9 +12217,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.40.0"
|
||||
version = "0.40.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "110e37d98880fd804cd47509b90a8444f7aaaab655ae625e83cd442a8bc2ebf2"
|
||||
checksum = "0e8a3edbe0394ce115f3917a00349dd242be09eb5a091ed509eaf67b55cfb8a8"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
@@ -12710,9 +12710,9 @@ checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff"
|
||||
checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -12743,9 +12743,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-build"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c07bd4e9f99e2a3ed8bebad596ae02a94e53745b68acc6900e22c3475c8bc44"
|
||||
checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4"
|
||||
dependencies = [
|
||||
"prettyplease 0.2.37",
|
||||
"proc-macro2",
|
||||
@@ -12755,9 +12755,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6"
|
||||
checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost 0.14.4",
|
||||
@@ -12766,9 +12766,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost-build"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a05d8957fac6b3542999796cb4449d45da790751962f974a457e5e00c43d78c"
|
||||
checksum = "2ea216ddc448e08e7839ab09e20e60b06b6a1c32745ccb212c0ebd161c199f1b"
|
||||
dependencies = [
|
||||
"prettyplease 0.2.37",
|
||||
"proc-macro2",
|
||||
@@ -13161,9 +13161,12 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0"
|
||||
dependencies = [
|
||||
"utf_types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
@@ -13171,6 +13174,12 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "utf_types"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.29.0"
|
||||
@@ -13285,12 +13294,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
|
||||
checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
@@ -13298,9 +13306,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.79"
|
||||
version = "0.4.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e"
|
||||
checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"tokio",
|
||||
@@ -13309,9 +13317,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
|
||||
checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -13319,9 +13327,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
|
||||
checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
@@ -13332,9 +13340,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
|
||||
checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -13354,9 +13362,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.106"
|
||||
version = "0.3.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4"
|
||||
checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
||||
+6
-6
@@ -175,9 +175,9 @@ tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -319,7 +319,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -334,7 +334,7 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.0"
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
@@ -386,7 +386,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
hotpath = { version = "0.30.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -16,8 +16,193 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -1636,20 +1636,53 @@ async fn hot_window_with_old_followers(
|
||||
let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
|
||||
let scheduler_config = run.assert_default_cadence().await?;
|
||||
let baseline = run.quiet_baseline().await?;
|
||||
let leader = run.leader().await?;
|
||||
let leader_pid = run.pid(leader)?;
|
||||
// A leader-lock refresh gap during old-binary replacement can make
|
||||
// `leader()` briefly observe a different node. Poll until the coordinator
|
||||
// is stable before comparing against the measured fleet. A persistent
|
||||
// change to an old-binary node is a genuine election outcome and fails.
|
||||
let (leader, leader_pid) = if mixed {
|
||||
let expected_leader = usize::try_from(
|
||||
mixed_fleet["leader"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader")?,
|
||||
)?;
|
||||
let expected_pid = mixed_fleet["leader_pid"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader pid")?;
|
||||
let mut resolved = None;
|
||||
for _ in 0..20 {
|
||||
let candidate = run.leader().await?;
|
||||
let candidate_pid = u64::from(run.pid(candidate)?);
|
||||
if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
|
||||
resolved = Some((candidate, run.pid(candidate)?));
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
match resolved {
|
||||
Some(pair) => pair,
|
||||
None => {
|
||||
let current = match run.leader().await {
|
||||
Ok(node) => Some((node, run.pid(node).ok())),
|
||||
Err(_) => None,
|
||||
};
|
||||
return Err(format!(
|
||||
"the hot window did not retain its measured new coordinator after stabilization \
|
||||
(expected node {expected_leader} pid {expected_pid}, observed {current:?})"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let leader = run.leader().await?;
|
||||
(leader, run.pid(leader)?)
|
||||
};
|
||||
let producer = (leader + 1) % 4;
|
||||
let producers = if fault.is_some() {
|
||||
vec![producer, (leader + 2) % 4]
|
||||
} else {
|
||||
vec![producer]
|
||||
};
|
||||
if mixed
|
||||
&& (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
|
||||
|| mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
|
||||
{
|
||||
return Err("the hot window did not retain its measured new coordinator".into());
|
||||
}
|
||||
let producer_pid = run.pid(producer)?;
|
||||
let producer_address = run.peer_address(producer);
|
||||
let [cold_a, cold_b, hot] = run.buckets.clone();
|
||||
|
||||
@@ -576,6 +576,41 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata_short(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_pool_meta_cas(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -2146,3 +2146,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -1112,8 +1112,9 @@ async fn wait_for_upgrade_writes(
|
||||
match response {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|
||||
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
|
||||
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|
||||
{
|
||||
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
|
||||
}
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -1901,6 +1902,15 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
|
||||
}
|
||||
@@ -5200,6 +5210,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
|
||||
@@ -747,6 +747,16 @@ async fn list_path_raw_inner(
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
if !revjob_rx.is_cancelled() {
|
||||
for (i, reader) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_none()
|
||||
&& pending_entries[i].is_none()
|
||||
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
|
||||
{
|
||||
pending_entries[i] = Some(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
@@ -878,7 +888,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
if entry.name == current.name {
|
||||
// If exact match, we agree.
|
||||
if current.matches_strict(&entry) {
|
||||
if current.matches_strict_borrowed(&entry) {
|
||||
top_entries[i] = Some(entry);
|
||||
agree += 1;
|
||||
} else {
|
||||
@@ -986,7 +996,9 @@ async fn list_path_raw_inner(
|
||||
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
let _ = r.skip(1).await;
|
||||
if !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
@@ -1009,7 +1021,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if top_entries[i].is_some() {
|
||||
if top_entries[i].is_some() && !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,14 +67,15 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
|
||||
MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -2452,6 +2453,239 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
/// Prepare a remote token without retrying the RPC. It owns the bucket
|
||||
/// publication fence but does not enter the object metadata queue yet.
|
||||
pub async fn reserve_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Use the distinct short-lease RPC; an old peer rejects it before allocating a token.
|
||||
pub(crate) async fn reserve_object_metadata_short(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
self.reserve_object_metadata_with_protocol(bucket, object, incarnation, timeout, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata_with_protocol(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
short: bool,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"reserve_object_metadata",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let result = if short {
|
||||
client.reserve_object_metadata_short(request).await
|
||||
} else {
|
||||
client.reserve_object_metadata(request).await
|
||||
};
|
||||
let response = result
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
if response.protocol_version != if short { 3 } else { 2 } {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
Uuid::from_slice(&response.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or(DiskError::Unexpected)
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
|
||||
/// result before allowing any subsequent unprotected mutation.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
self.update_object_metadata_reservation(bucket, object, token, timeout, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
self.update_object_metadata_reservation(bucket, object, token, timeout, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_object_metadata_reservation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: Duration,
|
||||
renew_only: bool,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"activate_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationActivateRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
renew_only,
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation activation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.activate_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed remote reservation. False cannot prove that an
|
||||
/// earlier claim did not start a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"cancel_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationCancelRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.cancel_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(response.success)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Publish a staged user-object write using a remote one-shot reservation.
|
||||
/// The caller must retain the bucket lifecycle fence until the tail drains.
|
||||
pub async fn rename_data_with_metadata_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
token: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_reserved_data",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let rename = RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: Bytes::new(),
|
||||
};
|
||||
let mut request = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(rename),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reserved rename request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.rename_reserved_data(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -5109,6 +5343,84 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_metadata_reservation_rejects_old_peer_without_legacy_rpc() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata_short("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec!["/node_service.NodeService/ReserveObjectMetadataShort"],
|
||||
"an old peer must never receive the longer-lived v2 request"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let token = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.renew_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote.cancel_object_metadata_reservation(token).await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.rename_data_with_metadata_reservation(
|
||||
crate::disk::RUSTFS_META_TMP_BUCKET,
|
||||
"staged",
|
||||
&sample_rename_data_file_info(),
|
||||
"bucket",
|
||||
"object",
|
||||
token,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec![
|
||||
"/node_service.NodeService/ReserveObjectMetadata",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/CancelObjectMetadataReservation",
|
||||
"/node_service.NodeService/RenameReservedData",
|
||||
],
|
||||
"reservation operations must never downgrade to legacy methods",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
@@ -1315,6 +1315,122 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
if !response.success {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
|
||||
Some(":0") => Some(false),
|
||||
Some(":1") => Some(true),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
|
||||
@@ -3506,6 +3506,7 @@ mod tests {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3720,33 +3720,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3756,8 +3768,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5822,13 +5834,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -28353,7 +28365,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28387,6 +28399,88 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28429,7 +28523,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28838,17 +28932,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28873,6 +28967,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -655,6 +655,19 @@ impl Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -604,8 +604,69 @@ impl LocalDiskWrapper {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
metadata_reservation_token,
|
||||
..
|
||||
} = guards;
|
||||
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
|
||||
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
|
||||
(Some(reservation), None) => Some(reservation),
|
||||
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
// A missing token may already have been claimed by an earlier
|
||||
// request that is still publishing; do not claim it was safe.
|
||||
Err(DiskError::DiskOngoingReq) => {
|
||||
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
},
|
||||
(None, None) => None,
|
||||
};
|
||||
// A queued reservation must acquire its metadata lease before disk
|
||||
// admission. Otherwise a waiting background mutation could hold the
|
||||
// disk permit needed by the earlier lease owner.
|
||||
let metadata_reservation = match metadata_reservation {
|
||||
Some(reservation) => Some(reservation.wait().await),
|
||||
None => None,
|
||||
};
|
||||
// The claimed reservation stays in this cancellable future while disk
|
||||
// admission can wait. Only physical work moves into an owned task.
|
||||
let prechecked_permit = if metadata_reservation.is_some() {
|
||||
// Background admission is exclusive. Waiting for it while holding
|
||||
// this object's lease can deadlock an admitted mutation waiting for
|
||||
// the same lease. An already owned permit does not need to wait.
|
||||
if current_disk_io_context().is_some_and(|context| {
|
||||
context.class == DiskIoClass::Background
|
||||
&& !context
|
||||
.permit
|
||||
.as_ref()
|
||||
.is_some_and(|permit| permit.is_for(&self.io_admission))
|
||||
}) {
|
||||
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let preflight = self
|
||||
.with_native_resources(async {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.acquire_disk_io_permit().await
|
||||
})
|
||||
.await;
|
||||
match preflight {
|
||||
Ok(Ok(Some(permit))) => Some(permit),
|
||||
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The reserved metadata lease must outlive a cancelled RPC waiter.
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
|
||||
Some(Arc::new(external_guard))
|
||||
} else {
|
||||
external_guard
|
||||
};
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -624,27 +685,54 @@ impl LocalDiskWrapper {
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let result = operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
let rename = || async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = if let Some(reservation) = metadata_reservation {
|
||||
Box::pin(operation.disk.rename_data_observed_with_reservation(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
(namespace_owner, Some(reservation)),
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await
|
||||
};
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
};
|
||||
let result = if let Some(permit) = prechecked_permit {
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
operation
|
||||
.with_native_resources(operation.track_admitted_disk_operation(
|
||||
"rename_data",
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
rename().await
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::Write,
|
||||
true,
|
||||
))
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
.await?
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
operation
|
||||
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
|
||||
.await
|
||||
};
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
@@ -1598,6 +1686,81 @@ pub struct LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
/// Reserve the metadata mutation lock for one object on this disk instance.
|
||||
/// Unclaimed reservations expire after 60 seconds.
|
||||
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.reserve_object_metadata(bucket, object, timeout).await
|
||||
}
|
||||
|
||||
/// Register an object metadata reservation in FIFO order on this disk.
|
||||
/// The returned token may be claimed before its physical lease is available.
|
||||
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.enqueue_object_metadata(bucket, object).await
|
||||
}
|
||||
|
||||
/// Prepare an inert reservation token bound to this disk and object.
|
||||
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
self.prepare_object_metadata_with_owner(bucket, object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner_ttl(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
ttl: Duration,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
|
||||
}
|
||||
|
||||
/// Refresh an inert token while the caller is still staging the write.
|
||||
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.renew_object_metadata(bucket, object, token)
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
|
||||
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
|
||||
self.disk.cancel_object_metadata_reservation(token)
|
||||
}
|
||||
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
@@ -2398,6 +2561,10 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2577,6 +2744,7 @@ impl LocalDiskWrapper {
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
@@ -2588,6 +2756,7 @@ impl LocalDiskWrapper {
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
identity_prechecked: bool,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
@@ -2609,7 +2778,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if let Err(err) = self.check_disk_stale().await {
|
||||
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return Err(err);
|
||||
}
|
||||
@@ -5577,6 +5746,115 @@ mod tests {
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
disk.make_volume("bucket").await.expect("bucket volume");
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
|
||||
disk.format_info.write().await.last_check = None;
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
disk.install_format_probe_hook(false, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(30));
|
||||
});
|
||||
let publisher = Arc::clone(&wrapper);
|
||||
let rename = tokio::spawn(async move {
|
||||
publisher
|
||||
.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&FileInfo::default(),
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), entered_rx)
|
||||
.await
|
||||
.expect("disk identity preflight starts")
|
||||
.expect("preflight hook");
|
||||
rename.abort();
|
||||
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
|
||||
let key = disk.get_object_path("bucket", "object").expect("object path");
|
||||
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
|
||||
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
|
||||
release_tx.send(()).expect("release disk identity preflight");
|
||||
if !released_before_io {
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), successor)
|
||||
.await
|
||||
.expect("preflight finishes"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
released_before_io,
|
||||
"a cancelled request must not retain the reservation before physical mutation starts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
let held = wrapper
|
||||
.io_admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("background permit");
|
||||
let fi = FileInfo::default();
|
||||
let rename = crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
wrapper.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&fi,
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
);
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
|
||||
drop(held);
|
||||
let observation = result.expect("reserved mutation must not wait while holding the object lease");
|
||||
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
|
||||
assert!(observation.rejected_before_publication());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
+2120
-204
File diff suppressed because it is too large
Load Diff
@@ -175,17 +175,30 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
old_metadata: &[u8],
|
||||
sync: bool,
|
||||
) -> std::io::Result<PathBuf> {
|
||||
let Some(staging_parent) = staging_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
|
||||
};
|
||||
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
|
||||
&& let Err(err) = std::fs::write(&backup_path, old_metadata)
|
||||
{
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -356,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(RenameDataResp),
|
||||
Committed(Box<RenameDataResp>),
|
||||
CommitErr(DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
#[cfg(test)]
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
@@ -428,12 +444,29 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
impl LocalRenamePreflightRejection {
|
||||
pub(in crate::disk) fn not_dispatched() -> Self {
|
||||
Self(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RenameDataState {
|
||||
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
|
||||
Self {
|
||||
metadata_reservation: Some(reservation),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep optional native MPU state machines out of ordinary rename frames.
|
||||
// Construct the future here as well as pinning it, so its temporary stays off
|
||||
// the caller's poll stack. Call only from the existing native MPU branches.
|
||||
@@ -475,6 +508,15 @@ impl LocalDisk {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// Pre-acquired metadata must never reverse quota or multipart-source
|
||||
// lock order. These paths need their own reservation protocol.
|
||||
if state.metadata_reservation.is_some()
|
||||
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
|
||||
{
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
|
||||
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
@@ -540,7 +582,9 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
|
||||
Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
|
||||
Some(Arc::new(
|
||||
pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -568,7 +612,27 @@ impl LocalDisk {
|
||||
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
|
||||
None => state.namespace_owner.take(),
|
||||
};
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
|
||||
// A reserved lease is single-use and bound to the exact disk path.
|
||||
// Keep the independent namespace/source owner alongside it; a pre-held
|
||||
// mutex cannot be acquired again without deadlocking.
|
||||
let reservation_fence = state
|
||||
.metadata_reservation
|
||||
.as_ref()
|
||||
.and_then(|reservation| reservation.fence());
|
||||
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
|
||||
if multipart_staging.is_some() || quota_fence_claim.is_some() {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let lease = reservation
|
||||
.into_lease_for(&root_destination_object_path, self.reservation_instance)
|
||||
.inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
Arc::new((lease, metadata_owner))
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
|
||||
};
|
||||
drop(metadata_wait);
|
||||
#[cfg(unix)]
|
||||
let parent_receipt = if native_timing {
|
||||
@@ -582,17 +646,23 @@ impl LocalDisk {
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let mut receipt_mutation = native_timing
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
|
||||
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&root_destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_lease),
|
||||
Some(metadata_owner),
|
||||
)
|
||||
.await;
|
||||
drop(namespace_wait);
|
||||
// The volume read lease prevents deletion of this disk's bucket until
|
||||
// publication finishes. Recheck after its wait, before any mutation.
|
||||
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -741,11 +811,6 @@ impl LocalDisk {
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -785,10 +850,11 @@ impl LocalDisk {
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
)?;
|
||||
// An existing xl.meta can share versions or require rollback
|
||||
// restoration when a post-rename directory sync fails.
|
||||
if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
// An existing xl.meta that replaces an existing version slot
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -811,7 +877,19 @@ impl LocalDisk {
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
if no_inline
|
||||
&& sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
@@ -830,13 +908,31 @@ impl LocalDisk {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& (sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
@@ -846,49 +942,157 @@ impl LocalDisk {
|
||||
if let Err(err) = sync_result
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
|
||||
}
|
||||
|
||||
let guard = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
guard,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
std::fs::remove_file(&src_file_path_owned)?;
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if has_data_dir_path_owned.is_some()
|
||||
&& let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}))
|
||||
{
|
||||
return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) =
|
||||
fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
|
||||
{
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
return Err(err);
|
||||
let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
destination_was_absent,
|
||||
staged_rollback_path.as_ref(),
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline {
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path_owned.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
@@ -897,7 +1101,7 @@ impl LocalDisk {
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))
|
||||
})))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
@@ -917,7 +1121,23 @@ impl LocalDisk {
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(resp);
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -925,11 +1145,269 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
|
||||
std::panic::resume_unwind(panic_payload);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = restored_fi;
|
||||
fi = *restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else if let Some(publication) = multipart_publication.as_ref()
|
||||
&& no_inline
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
AbortBeforeDataRename(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
RestoreTargetAfterError(DiskError, bool),
|
||||
}
|
||||
let publication_owned = Arc::clone(publication);
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let outcome = os::run_blocking_namespace_operation_with_sync(
|
||||
mutation_lease.clone(),
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let skip_parent_owned = if has_dst_buf_owned.is_some() {
|
||||
dst_parent_owned.clone()
|
||||
} else {
|
||||
dst_volume_dir_owned.clone()
|
||||
};
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf_owned.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
if let Err(err) = xlmeta.add_version(fi_moved) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
|
||||
}
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf_owned = match xlmeta.marshal_msg() {
|
||||
Ok(buf) => buf,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
|
||||
};
|
||||
|
||||
let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
|
||||
&new_dst_buf_owned,
|
||||
has_dst_buf_owned.as_deref(),
|
||||
&dst_file_path_owned,
|
||||
&directory_guard_owned,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
// When presynced, shard payloads were synced during UploadPart and
|
||||
// prepare_and_install_head_on_thread just fsynced src_data_path after
|
||||
// writing RETIREMENT_ANCHOR.
|
||||
if sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
|
||||
&& let Err(err) = os::sync_dir_files_std(src_data_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
let directory_guard_owned = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
directory_guard_owned,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let Some(src_parent) = src_data_path.parent() else {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
|
||||
};
|
||||
let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
|
||||
directory_guard_owned,
|
||||
DiskError::Unexpected,
|
||||
));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent_owned,
|
||||
&publication_root_owned,
|
||||
&directory_guard_owned,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
|
||||
}
|
||||
if has_dst_buf_owned.is_none() {
|
||||
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir_owned) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir_limited_std(dir) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
|
||||
to_file_error(err).into(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if dir == dst_volume_dir_owned.as_path() {
|
||||
break;
|
||||
}
|
||||
ancestor = dir.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))),
|
||||
Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
match outcome {
|
||||
MultipartFusedCommitOutcome::Committed(resp) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
|
||||
MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
|
||||
if !invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
|
||||
if invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
@@ -1053,71 +1531,81 @@ impl LocalDisk {
|
||||
// after the commit rename, and a crash before the rename means the
|
||||
// PUT was never acknowledged. A metadata commit: relaxed tiers
|
||||
// leave it to the page cache.
|
||||
if let Some(publication) = multipart_publication.as_ref() {
|
||||
pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
|
||||
}
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
// The tmp xl.meta write and the shard-file fdatasync are independent
|
||||
// (disjoint paths) and both only need to be durable before the commit
|
||||
// renames below, so run them concurrently to drop a blocking
|
||||
// round-trip from the PUT commit critical path (rustfs/backlog#922
|
||||
// step 2). The "contents durable -> rename -> dst dir fsync" ordering
|
||||
// is unchanged — both futures complete before any rename — which the
|
||||
// rename_data crash-consistency harness (backlog#935) exercises.
|
||||
//
|
||||
// Shard durability: once rename_data succeeds the write is
|
||||
// acknowledged, so data must not live only in the page cache.
|
||||
// Multipart parts were already synced during rename_part, so their
|
||||
// fdatasync here is a cheap no-op. A missing source dir is left for the
|
||||
// rename below to report through the existing rollback path. Payload
|
||||
// durability is kept by both strict and relaxed.
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = multipart_publication
|
||||
.as_ref()
|
||||
.map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(all(not(test), target_os = "linux"))]
|
||||
let probe_active = false;
|
||||
let shard_sync = async {
|
||||
if durability.syncs_data_shards()
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
#[cfg(target_os = "linux")]
|
||||
if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
if multipart_publication.is_none()
|
||||
&& let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
// Surface a tmp-meta failure first (its prior serial position), then a
|
||||
// shard-sync failure; either aborts before any rename, exactly as the
|
||||
// sequential version did.
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
let prepared_metadata_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
|
||||
None => prepared_metadata_source,
|
||||
let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
|
||||
let dst_file_path_ref = &dst_file_path;
|
||||
let dir_guard_ref = &commit.directory_guard;
|
||||
let lease_clone = commit.mutation_lease.clone();
|
||||
let prepare_fut = pin_multipart_future(|| {
|
||||
publication.prepare_and_install_head(
|
||||
&new_dst_buf,
|
||||
has_dst_buf.as_deref(),
|
||||
dst_file_path_ref,
|
||||
dir_guard_ref,
|
||||
lease_clone,
|
||||
)
|
||||
});
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
} else {
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = commit.source_metadata.to_path_buf();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
};
|
||||
commit.directory_guard = remove_dst_base_before_commit(
|
||||
dst_path,
|
||||
@@ -1457,13 +1945,22 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
|
||||
return Err(error);
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
Ok(RenameDataResp {
|
||||
@@ -1663,13 +2160,8 @@ impl LocalDisk {
|
||||
#[cfg(not(windows))]
|
||||
let backup_path = {
|
||||
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata)?
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
if sync {
|
||||
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
@@ -2109,8 +2601,23 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
|
||||
) -> super::super::RenameDataObservation {
|
||||
let (namespace_owner, metadata_reservation) = owners;
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
..Default::default()
|
||||
};
|
||||
let result = self
|
||||
|
||||
@@ -1000,6 +1000,25 @@ pub(super) fn inspection_with_markers(
|
||||
Ok((inspection, has_physical_data_dirs_or_markers))
|
||||
}
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Native MPU source closure has its own conditional protocol. Generic
|
||||
// object recovery must never restore its head or acquire SourceRW under
|
||||
@@ -1523,6 +1542,9 @@ impl LocalDisk {
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
}
|
||||
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
.delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
|
||||
@@ -1911,8 +1933,14 @@ impl LocalDisk {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let mut file_leases = Vec::with_capacity(3);
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let mut file_leases = Vec::with_capacity(5);
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
let path = rollback.join(name);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
|
||||
file_leases.push((path, lease));
|
||||
@@ -2132,119 +2160,151 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let (old_dir, journal_raw) = match (|| -> Result<Option<(Option<Uuid>, Vec<u8>)>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let deleted_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| v.header.version_id.unwrap_or_default())
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
if !meta.versions.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut predecessors = BTreeSet::new();
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let Some(id) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| Uuid::parse_str(name).ok())
|
||||
.filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let prepared = regular_bytes(&entry.path().join(PREPARED))?;
|
||||
let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let surviving: HashSet<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
deleted_headers.retain(|header| !surviving.contains(header));
|
||||
let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let file_name = entry.file_name();
|
||||
let Some(name_str) = file_name.to_str() else {
|
||||
sole_transaction = false;
|
||||
continue;
|
||||
};
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let dir_path = entry.path();
|
||||
let prepared = regular_bytes(&dir_path.join(PREPARED))?;
|
||||
let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
if sole_transaction && name_str == id.hyphenated().to_string() && old_dir != Some(id) {
|
||||
let is_surviving_data_dir = original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& prepared.is_none()
|
||||
&& receipt.is_none()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
}
|
||||
}
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: None,
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
sole_transaction &= predecessors.is_empty();
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: target.as_deref().map(digest),
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
let raw = journal_bytes(&journal)?;
|
||||
Ok(Some((old_dir, target, raw, journal, sole_transaction)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
Ok(Some((old_dir, journal_bytes(&journal)?)))
|
||||
})() {
|
||||
Ok(Some(pair)) => pair,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = tempfile::Builder::new()
|
||||
.prefix("delete-preparation-")
|
||||
.tempdir_in(&staging_root)?;
|
||||
for (name, bytes) in [
|
||||
(STORAGE_FORMAT_FILE_BACKUP, buf.as_slice()),
|
||||
(PREPARED, journal_raw.as_slice()),
|
||||
] {
|
||||
let mut file = std::fs::File::create(staged.path().join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
|
||||
let staged_dir = staged.path().to_path_buf();
|
||||
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, &buf)?;
|
||||
if sync && !buf.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
}
|
||||
let mut written_files = Vec::with_capacity(2);
|
||||
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
|
||||
let Some(bytes) = bytes else {
|
||||
continue;
|
||||
};
|
||||
let mut file = std::fs::File::create(staged_dir.join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
written_files.push(file);
|
||||
}
|
||||
}
|
||||
for file in written_files {
|
||||
file.sync_data()?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(staged.path())?;
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
}
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
@@ -2257,27 +2317,62 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
let staged_dir = staged.keep();
|
||||
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
|
||||
let _ = std::fs::remove_dir_all(&staged_dir);
|
||||
return Err(err);
|
||||
}
|
||||
std::fs::rename(staged.path(), &destination)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&file_path)?;
|
||||
let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(uuid) = old_dir {
|
||||
let old_path = file_path.join(uuid.to_string());
|
||||
if old_path.is_dir() {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
if let Some(old_path) = old_data_path {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let reserve_res = (|| -> std::io::Result<()> {
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = reserve_res {
|
||||
let _ = std::fs::remove_file(&reserved_marker);
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
if let Some(ref target_bytes) = target {
|
||||
let target_in_rollback = destination.join(TARGET);
|
||||
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
|
||||
let rename_res = std::fs::rename(&linked_xl, &xl_path);
|
||||
if rename_res.is_err() {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
}
|
||||
rename_res
|
||||
} else {
|
||||
(|| -> std::io::Result<()> {
|
||||
let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
|
||||
if sync && !target_bytes.is_empty() {
|
||||
staged_xl.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_xl.path(), &xl_path)
|
||||
})()
|
||||
};
|
||||
if let Err(err) = write_res {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
} else if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
@@ -2298,6 +2393,7 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
put_prepared_delete_receipt(destination, (journal, sole_transaction));
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
@@ -2343,6 +2439,26 @@ impl LocalDisk {
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
|
||||
&& rollback.join(PREPARED).is_file()
|
||||
{
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: false,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged.path(), &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
return Ok(Ok(Some((journal, sole_transaction, true))));
|
||||
}
|
||||
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
|
||||
Ok(Some(bytes)) => bytes,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
@@ -2360,12 +2476,18 @@ impl LocalDisk {
|
||||
{
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let current_exists = match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
let target_satisfied = match journal.target_digest {
|
||||
None => match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => !exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
},
|
||||
Some(expected_target) => match regular_bytes(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(Some(bytes)) => digest(&bytes) == expected_target,
|
||||
Ok(None) => false,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
},
|
||||
};
|
||||
let unversioned_target_absent = journal.target_digest.is_none() && !current_exists;
|
||||
if !unversioned_target_absent {
|
||||
if !target_satisfied {
|
||||
return Ok(Ok(None));
|
||||
}
|
||||
let mut sole_transaction = journal.predecessors.is_empty();
|
||||
@@ -2382,8 +2504,15 @@ impl LocalDisk {
|
||||
&& id != transaction
|
||||
&& !journal.data_dirs.contains(&id)
|
||||
{
|
||||
sole_transaction = false;
|
||||
break;
|
||||
let dir_path = entry.path();
|
||||
let is_surviving_data_dir = journal.original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& !dir_path.join(PREPARED).exists()
|
||||
&& !dir_path.join(DELETE_QUORUM_COMMITTED_FILE).exists()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2406,7 +2535,7 @@ impl LocalDisk {
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
Ok(Ok(Some((journal, sole_transaction, unversioned_target_absent))))
|
||||
Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2458,12 +2587,15 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let has_target = journal.target_digest.is_some();
|
||||
let journal_for_receipt = journal.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
@@ -2474,11 +2606,13 @@ impl LocalDisk {
|
||||
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let mut had_physical_data_dirs = false;
|
||||
for data_dir in &data_dirs {
|
||||
for base in [&root, &rollback] {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in entries {
|
||||
@@ -2519,23 +2653,45 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !data_dirs.is_empty() && sync && root.is_dir() {
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
}
|
||||
if has_target || had_physical_data_dirs {
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal_for_receipt,
|
||||
finished: true,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged_receipt.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_receipt.path(), &receipt_path)?;
|
||||
let _ = std::fs::File::create(rollback.join(FINISHED))?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
}
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
|
||||
ignore_not_found(std::fs::remove_file(&receipt_path))?;
|
||||
for dir_path in &prune_paths {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
|
||||
if let Err(err) = std::fs::remove_dir(dir_path) {
|
||||
if is_dir_not_empty_error(&err) {
|
||||
break;
|
||||
@@ -6282,6 +6438,214 @@ mod tests {
|
||||
b"outside bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_marker_and_publish_fast_path() {
|
||||
let (_temp, disk, info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object root");
|
||||
let transaction = Uuid::new_v4();
|
||||
let mut marker = FileInfo {
|
||||
name: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: info.mod_time.map(|t| t + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &marker, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused delete marker preparation must succeed");
|
||||
assert_eq!(fused, Some(()), "versioned delete marker must use fused path");
|
||||
assert!(root.join(transaction.to_string()).join(TARGET).is_file());
|
||||
assert!(root.join(STORAGE_FORMAT_FILE).is_file());
|
||||
|
||||
let published = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("versioned target must satisfy local publish");
|
||||
assert!(published.1, "sole transaction must be true");
|
||||
assert!(published.2, "target_satisfied must be true");
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &published.0, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
let final_meta = FileMeta::load(&fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("read final xl.meta"))
|
||||
.expect("decode final xl.meta");
|
||||
assert_eq!(final_meta.versions.len(), 2);
|
||||
assert_eq!(final_meta.versions[0].header.version_id, marker.version_id);
|
||||
marker.deleted = false;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
|
||||
let (_temp, disk, mut v1, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
// Configure v1 and v2 with distinct physical data directories so deleting v2
|
||||
// exercises both `old_dir` reservation and `is_surviving_data_dir`.
|
||||
v1.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut v1.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let v1_dir = Uuid::new_v4();
|
||||
v1.data_dir = Some(v1_dir);
|
||||
fs::create_dir_all(root.join(v1_dir.to_string()))
|
||||
.await
|
||||
.expect("create v1 dir");
|
||||
fs::write(root.join(v1_dir.to_string()).join("part.1"), b"v1-data")
|
||||
.await
|
||||
.expect("write v1 part");
|
||||
|
||||
let mut v2 = v1.clone();
|
||||
v2.version_id = Some(Uuid::new_v4());
|
||||
let v2_dir = Uuid::new_v4();
|
||||
v2.data_dir = Some(v2_dir);
|
||||
fs::create_dir_all(root.join(v2_dir.to_string()))
|
||||
.await
|
||||
.expect("create v2 dir");
|
||||
fs::write(root.join(v2_dir.to_string()).join("part.1"), b"v2-data")
|
||||
.await
|
||||
.expect("write v2 part");
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(v1.clone()).expect("add v1");
|
||||
meta.add_version(v2.clone()).expect("add v2");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused versioned delete preparation must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
assert!(
|
||||
root.join(v2_dir.to_string())
|
||||
.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"))
|
||||
.is_file(),
|
||||
"deleted version data_dir must have recovery marker"
|
||||
);
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("versioned target must satisfy local publish");
|
||||
assert!(sole_transaction, "surviving v1 data_dir must not clear sole_transaction");
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_some());
|
||||
|
||||
// Simulate interruption during finish-before-finalize right after TARGET is removed
|
||||
// (after finished receipt + FINISHED marker were persisted, before receipt_path is removed).
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: true,
|
||||
};
|
||||
fs::write(
|
||||
rollback.join(DELETE_QUORUM_COMMITTED_FILE),
|
||||
serde_json::to_vec(&finished_receipt).expect("serialize finished receipt"),
|
||||
)
|
||||
.await
|
||||
.expect("write finished receipt");
|
||||
fs::write(rollback.join(FINISHED), []).await.expect("write FINISHED");
|
||||
fs::remove_file(rollback.join(STORAGE_FORMAT_FILE_BACKUP))
|
||||
.await
|
||||
.expect("remove backup");
|
||||
fs::remove_file(rollback.join(TARGET)).await.expect("remove TARGET");
|
||||
|
||||
let inspected = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("interrupted post-TARGET state must remain inspectable without FileCorrupt");
|
||||
assert_eq!(inspected.state, DeleteRecoveryState::CommitFinished);
|
||||
|
||||
// Complete cleanup via try_finish_and_finalize_prepared_delete_local.
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed after interruption");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(v2_dir.to_string()).exists(), "deleted v2 data_dir must be removed");
|
||||
assert!(
|
||||
root.join(v1_dir.to_string()).join("part.1").is_file(),
|
||||
"surviving v1 data_dir must remain intact"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
|
||||
let (_temp, disk, mut info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
info.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut info.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let data_dir = Uuid::new_v4();
|
||||
info.data_dir = Some(data_dir);
|
||||
fs::create_dir_all(root.join(data_dir.to_string()))
|
||||
.await
|
||||
.expect("create data_dir");
|
||||
fs::write(root.join(data_dir.to_string()).join("part.1"), b"physical-payload")
|
||||
.await
|
||||
.expect("write part.1");
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(info.clone()).expect("add version");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &info, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused unversioned physical delete must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("absent target must satisfy local publish");
|
||||
assert!(sole_transaction);
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_none());
|
||||
|
||||
let rollback = root.join(transaction.to_string());
|
||||
assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
|
||||
assert!(!root.exists(), "empty object directory must be pruned");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -421,6 +421,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
|
||||
super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
|
||||
let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
let source = store.source.clone();
|
||||
let base = store.base.clone();
|
||||
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
root: os::PublicationRoot,
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
|
||||
@@ -238,9 +238,192 @@ pub(super) fn read_optional_in_pinned(
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct FileStamp {
|
||||
len: u64,
|
||||
modified: Option<std::time::SystemTime>,
|
||||
#[cfg(unix)]
|
||||
ino: u64,
|
||||
}
|
||||
|
||||
impl FileStamp {
|
||||
fn from_path(path: &Path) -> Option<Self> {
|
||||
let meta = std::fs::symlink_metadata(path).ok()?;
|
||||
Some(Self {
|
||||
len: meta.len(),
|
||||
modified: meta.modified().ok(),
|
||||
#[cfg(unix)]
|
||||
ino: {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
meta.ino()
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveMultipartSourceEntry {
|
||||
raw: Bytes,
|
||||
fi: FileInfo,
|
||||
head_stamp: FileStamp,
|
||||
unsettled_parts: std::collections::HashSet<usize>,
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
fi,
|
||||
head_stamp,
|
||||
unsettled_parts: std::collections::HashSet::new(),
|
||||
parts: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
source: &Path,
|
||||
part_number: Option<usize>,
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
map.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
|
||||
return None;
|
||||
}
|
||||
Some(part)
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
if !initial && has_active_multipart_source_head(source) {
|
||||
return Ok(());
|
||||
}
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
@@ -440,6 +623,7 @@ impl SourceStore {
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
@@ -470,7 +654,7 @@ impl SourceStore {
|
||||
let terminal = terminal_record_path(&source, state);
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
|
||||
Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
@@ -539,7 +723,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -551,6 +735,7 @@ impl SourceStore {
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
@@ -582,9 +767,10 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
@@ -1097,6 +1283,82 @@ pub(super) fn load_source(
|
||||
})
|
||||
}
|
||||
|
||||
fn load_uninitialized_source(
|
||||
raw: Bytes,
|
||||
path: &str,
|
||||
disk_id: Uuid,
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let sha = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let current = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
native_source_protocol(¤t)?;
|
||||
if source_status(¤t)?.is_some()
|
||||
|| publication_identity(¤t)?.is_some()
|
||||
|| SourceConditionV1::take(&mut current.clone())?.is_some()
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-bucket")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let object = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-object")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let condition = capture_source_condition(bucket, object, path, ¤t, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
|
||||
if condition.generation.epoch != 0 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let slot = condition
|
||||
.lineage
|
||||
.geometry
|
||||
.distribution
|
||||
.iter()
|
||||
.position(|index| *index == current.erasure.index)
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
|
||||
let snapshot = Some(MultipartSourceSnapshot {
|
||||
disk_id,
|
||||
physical_slot: slot,
|
||||
lineage: condition.lineage,
|
||||
generation: condition.generation,
|
||||
revision: 0,
|
||||
phase: MultipartSourcePhase::Active,
|
||||
state_sha256: [0; 32],
|
||||
head_sha256: Some(sha),
|
||||
inventory_sha256: inventory,
|
||||
capability_sha256: None,
|
||||
publication: None,
|
||||
});
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state: None,
|
||||
head: Some(SourceHead {
|
||||
sha256: sha,
|
||||
length,
|
||||
raw: Some(raw.clone()),
|
||||
}),
|
||||
original: Some(raw),
|
||||
info: Some(current),
|
||||
snapshot,
|
||||
head_matches_state: true,
|
||||
head_matches_activation: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -1323,41 +1585,6 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
/// Only native source validation may create provenance. Ordinary TMP/heal
|
||||
/// publication can retain a valid old local anchor, never issue/rebind one.
|
||||
pub async fn write_retirement_anchor(
|
||||
&self,
|
||||
state: &SourceState,
|
||||
publication: &PublicationIdentity,
|
||||
directory: &Path,
|
||||
lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
if state.lineage != publication.lineage
|
||||
|| state.generation != publication.source_generation
|
||||
|| state.disk_id != self.disk_id
|
||||
|| state.physical_slot != self.physical_slot
|
||||
{
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: self.disk_id,
|
||||
physical_slot: self.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let bytes = encode(&record)?;
|
||||
let directory = directory.to_path_buf();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
/// This caller already owns SourceRW and destination metadata ownership.
|
||||
/// An Aborting survivor can confirm a historical target only with its
|
||||
/// independent, same-disk publication anchor. Closure never creates one.
|
||||
@@ -1456,6 +1683,8 @@ struct ValidatedPreparedPublication {
|
||||
length: u64,
|
||||
}
|
||||
|
||||
type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
|
||||
|
||||
pub(super) struct NativeMultipartPublication {
|
||||
pub store: Arc<SourceStore>,
|
||||
pub publication: PublicationIdentity,
|
||||
@@ -1463,22 +1692,49 @@ pub(super) struct NativeMultipartPublication {
|
||||
state: parking_lot::Mutex<SourceState>,
|
||||
capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
|
||||
validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
|
||||
deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
pub(super) async fn prepare_native_multipart_publication(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_native_multipart_publication_deferred(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_native_multipart_publication_inner(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
defer_persist: bool,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
// Real Complete is parts-backed, including a zero-byte last part.
|
||||
if incoming.data.is_some() || incoming.inline_data() {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
|
||||
let loaded = store.load().await?;
|
||||
let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
|
||||
load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
|
||||
} else {
|
||||
store.load().await?
|
||||
};
|
||||
let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
let original = loaded
|
||||
.head
|
||||
@@ -1569,7 +1825,44 @@ impl LocalDisk {
|
||||
};
|
||||
// Copy the small source FI to an independent immutable inode. Its
|
||||
// canonical inode is never truncated to build the destination proxy.
|
||||
store.write_owned_backup_and_persist(owner, false, original, &state).await?;
|
||||
let directory = staging.private_directory(publication.final_data_dir);
|
||||
let anchor_record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: store.disk_id,
|
||||
physical_slot: store.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let anchor_bytes = encode(&anchor_record)?;
|
||||
state.validate(store.disk_id, &store.path)?;
|
||||
let original_bytes = original.to_vec();
|
||||
let deferred_initial = if defer_persist {
|
||||
Some((original_bytes, directory, anchor_bytes))
|
||||
} else {
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source_dir)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
None
|
||||
};
|
||||
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
|
||||
let context = NativeMultipartPublication {
|
||||
store,
|
||||
@@ -1578,14 +1871,9 @@ impl LocalDisk {
|
||||
state: parking_lot::Mutex::new(state),
|
||||
capability: parking_lot::Mutex::new(Some(capability)),
|
||||
validated_prepared: parking_lot::Mutex::new(None),
|
||||
deferred_initial: parking_lot::Mutex::new(deferred_initial),
|
||||
legacy,
|
||||
};
|
||||
let directory = staging.private_directory(context.publication.final_data_dir);
|
||||
let reserved = context.state.lock().clone();
|
||||
context
|
||||
.store
|
||||
.write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
|
||||
.await?;
|
||||
Ok(context)
|
||||
}
|
||||
}
|
||||
@@ -1594,18 +1882,15 @@ impl NativeMultipartPublication {
|
||||
pub fn canonical_head(&self) -> PathBuf {
|
||||
self.store.source.join(STORAGE_FORMAT_FILE)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if encode(current_state)? != encode(&state)? || !current.head_matches_state {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
// Validate the actual persisted representation once, after FileMeta's
|
||||
// canonicalization. No unvalidated input can install a source proxy.
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
@@ -1618,17 +1903,297 @@ impl NativeMultipartPublication {
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
self.store
|
||||
.write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
|
||||
.await?;
|
||||
} else {
|
||||
self.store.persist(&state).await?;
|
||||
}
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
let prepared_path = &self.prepared_path;
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
|
||||
if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = setup_res {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub async fn prepare_and_install_head(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
let prepared_path = self.prepared_path.clone();
|
||||
let canonical = self.canonical_head();
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(&prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
|
||||
prepared_metadata_source.write_all(&new_bytes, true)?;
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub async fn prepare_and_install_head(
|
||||
self: &Arc<Self>,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let this = Arc::clone(self);
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
|
||||
{
|
||||
let guard = self.validated_prepared.lock();
|
||||
@@ -1699,13 +2264,13 @@ impl NativeMultipartPublication {
|
||||
// return an error after consuming the last claimable source heads.
|
||||
self.store.restore_exact_target_with_lease(&state, lease).await
|
||||
}
|
||||
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if current.head.is_some() || encode(current_state)? != encode(&state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let (expected_sha256, expected_length) = {
|
||||
let guard = self.validated_prepared.lock();
|
||||
let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
@@ -1717,16 +2282,88 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
(validated.sha256, validated.length)
|
||||
};
|
||||
self.store
|
||||
.read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
let target_dir = self.store.target_path(&self.publication)?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
let snapshot = self.store.persist_with_source_sync(&state, true).await?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleaned.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
let proof = self
|
||||
.store
|
||||
@@ -1738,6 +2375,21 @@ impl NativeMultipartPublication {
|
||||
undo_capability: capability,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finish(
|
||||
self: &Arc<Self>,
|
||||
target_lease: Arc<os::NamespaceMutationLease>,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
|
||||
@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
|
||||
initial: bool,
|
||||
}
|
||||
impl MultipartSourceAdmission {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn is_initial(&self) -> bool {
|
||||
self.initial
|
||||
}
|
||||
pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
|
||||
let admission = self.clone();
|
||||
Arc::new(move || admission.validate())
|
||||
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
|
||||
logical_path: PathBuf,
|
||||
io_path: PathBuf,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
cleaned: Arc<std::sync::atomic::AtomicBool>,
|
||||
fresh_source_head: Option<bytes::Bytes>,
|
||||
}
|
||||
|
||||
impl MultipartCommitStaging {
|
||||
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
|
||||
pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
|
||||
self.io_path.join(directory.to_string())
|
||||
}
|
||||
pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
|
||||
(self.io_path.clone(), Arc::clone(&self.cleaned))
|
||||
}
|
||||
pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
|
||||
self.fresh_source_head.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MultipartCommitStaging {
|
||||
fn drop(&mut self) {
|
||||
if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let logical_path = self.logical_path.clone();
|
||||
let io_path = self.io_path.clone();
|
||||
let owner = self.owner.clone();
|
||||
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
let root = upload_scope_root(upload_or_part_path, part_path);
|
||||
let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
|
||||
return false;
|
||||
};
|
||||
super::multipart_source_state::has_clean_active_multipart_source(&source)
|
||||
}
|
||||
|
||||
pub(super) fn multipart_source_admission(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -356,12 +381,14 @@ impl LocalDisk {
|
||||
let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
|
||||
// The attempt root is exactly the namespace used by the tmp sweeper.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
|
||||
let staging = MultipartCommitStaging {
|
||||
let mut staging = MultipartCommitStaging {
|
||||
path: attempt,
|
||||
lease,
|
||||
logical_path,
|
||||
io_path,
|
||||
owner,
|
||||
cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
fresh_source_head: None,
|
||||
};
|
||||
let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
|
||||
let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
|
||||
@@ -371,31 +398,37 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
|
||||
let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
|
||||
(raw, source, true)
|
||||
} else {
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
(bytes::Bytes::from(raw), source, false)
|
||||
};
|
||||
let source_dir = source
|
||||
.data_dir
|
||||
.filter(|dir| !dir.is_nil())
|
||||
@@ -412,23 +445,31 @@ impl LocalDisk {
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut pinned_source_data = None;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
pinned_source_data =
|
||||
Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
|
||||
}
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
|
||||
};
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
.max(actual.actual_size);
|
||||
@@ -464,10 +505,15 @@ impl LocalDisk {
|
||||
cancelled,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
let has_state = !from_active_cache
|
||||
&& (source_object.join(".mpu-source-state").exists()
|
||||
|| source_object.join(".mpu-recovery").exists()
|
||||
|| source_object.join(".mpu-completions").exists());
|
||||
Ok((!has_state).then_some(raw))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
staging.fresh_source_head = fresh_head;
|
||||
Ok(staging)
|
||||
}
|
||||
}
|
||||
@@ -1219,4 +1265,211 @@ mod tests {
|
||||
.expect("non-part empty hash cleanup must remain supported");
|
||||
assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
|
||||
use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
|
||||
let (_root, disk, completed, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
|
||||
.await
|
||||
.expect("fresh multipart upload metadata");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
let read_fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("cached read_version");
|
||||
assert_eq!(read_fi.data_dir, Some(data_dir));
|
||||
|
||||
let destination = format!("{upload}/{data_dir}/part.1");
|
||||
let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part transaction");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"in-flight part transaction must mark active source as having unsettled part txn"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part transaction");
|
||||
assert!(
|
||||
disk.has_clean_active_multipart_source(&upload, false),
|
||||
"settled commit must restore clean active multipart source status"
|
||||
);
|
||||
let read_parts = disk
|
||||
.read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
|
||||
.await
|
||||
.expect("cached read_parts");
|
||||
assert_eq!(read_parts.len(), 1);
|
||||
assert_eq!(read_parts[0].number, 1);
|
||||
|
||||
// Prepare part 2, then replay Commit for already-settled part 1: part 2's
|
||||
// pending transaction must keep the upload marked as having unsettled part txns.
|
||||
let destination2 = format!("{upload}/{data_dir}/part.2");
|
||||
let mut part2_info = completed.parts[0].clone();
|
||||
part2_info.number = 2;
|
||||
let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part 2");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part 2 transaction");
|
||||
assert!(!disk.has_clean_active_multipart_source(&upload, false));
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replayed part 1 commit settlement");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"replayed settlement for part 1 must not clear part 2's pending transaction"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part 2");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part 2 transaction");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
// Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
|
||||
.await
|
||||
.expect("delete xl.meta via delete_paths");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"per-file head deletion must invalidate the canonical upload cache key"
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
|
||||
use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
|
||||
|
||||
let (_root, disk, _, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
let source_dir = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
|
||||
.expect("source dir");
|
||||
let base_dir = disk
|
||||
.io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.expect("multipart bucket path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
let init_wrapper = wrapper.clone();
|
||||
let init_upload = upload.clone();
|
||||
let init_task = tokio::spawn(async move {
|
||||
init_wrapper
|
||||
.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("initial write_metadata reaches parent dir fsync")
|
||||
.expect("entered channel");
|
||||
|
||||
let retire_wrapper = wrapper.clone();
|
||||
let retire_path = format!("{upload}/xl.meta");
|
||||
let mut retire_fut = Box::pin(async move {
|
||||
retire_wrapper
|
||||
.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
|
||||
"concurrent retirement must block on SourceRW while initial head publication is in flight"
|
||||
);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
init_task.await.expect("init join").expect("initial write_metadata succeeds");
|
||||
retire_fut
|
||||
.await
|
||||
.expect("concurrent retirement completes after SourceRW release");
|
||||
|
||||
assert!(
|
||||
!super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
|
||||
"retired source must not remain active in cache after interleaved initial publication"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
super::super::multipart_source_state::validate_source_mutation(
|
||||
&source_dir,
|
||||
&base_dir,
|
||||
&disk.publication_root,
|
||||
false
|
||||
),
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"validate_source_mutation must reject non-initial mutation on retired source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+119
-12
@@ -106,6 +106,8 @@ pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
pub(crate) metadata_reservation_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
@@ -122,6 +124,13 @@ impl RenameDataObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn not_dispatched(error: DiskError) -> Self {
|
||||
Self {
|
||||
result: Err(error),
|
||||
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
@@ -311,6 +320,82 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Prepare an inert single-use reservation on this disk.
|
||||
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
|
||||
pub async fn prepare_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.prepare_object_metadata(bucket, object),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh an inert token while its sender stages the write.
|
||||
pub async fn renew_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.renew_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token in object mutation order.
|
||||
pub async fn activate_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.activate_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. False does not prove that a claim has
|
||||
/// not already started a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
match self {
|
||||
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
|
||||
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
@@ -401,6 +486,13 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
|
||||
Self::Remote(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1244,18 +1336,33 @@ impl Disk {
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
Disk::Remote(remote_disk) => {
|
||||
if guards.metadata_reservation.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
if let Some(token) = guards.metadata_reservation_token {
|
||||
if guards.scanner_publication_lease_token.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
return RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+838
-119
File diff suppressed because it is too large
Load Diff
@@ -828,13 +828,27 @@ pub async fn create_bitrot_writer(
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
create_bitrot_writer_with_orig_volume(is_inline_buffer, disk, "", volume, path, length, shard_size, checksum_algo).await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn create_bitrot_writer_with_orig_volume(
|
||||
is_inline_buffer: bool,
|
||||
disk: Option<&DiskStore>,
|
||||
origvolume: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
let writer = if is_inline_buffer {
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else if let Some(disk) = disk {
|
||||
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
|
||||
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
let file = disk.create_file(origvolume, volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let file = instrument_raw_shard_writer(file, disk.is_local());
|
||||
CustomWriter::new_tokio_writer(file)
|
||||
|
||||
@@ -429,6 +429,7 @@ impl WritePlan {
|
||||
let checksum = reader.content_hash().clone();
|
||||
let trailer = reader.get_trailer().cloned();
|
||||
|
||||
let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
|
||||
let encrypted = self.encryption.is_some();
|
||||
if let Some(algorithm) = self.compression {
|
||||
reader = HashReader::from_reader(
|
||||
@@ -448,7 +449,8 @@ impl WritePlan {
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
|
||||
@@ -333,6 +333,34 @@ struct ProofCache {
|
||||
preferred: usize,
|
||||
}
|
||||
|
||||
type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
|
||||
|
||||
static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.get(&(part.generation, part.root, stripe))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
|
||||
let mut map = VERIFIED_STRIPE_RECORDS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().copied()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert((part.generation, part.root, stripe), record);
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.retain(|(stored_gen, _, _), _| *stored_gen != generation);
|
||||
}
|
||||
|
||||
impl PartProofReader {
|
||||
pub(crate) async fn reconstruction_proof(
|
||||
&self,
|
||||
@@ -410,6 +438,9 @@ impl PartProofReader {
|
||||
|
||||
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
|
||||
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
|
||||
if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let mut cached = self.cached.lock().await;
|
||||
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
|
||||
return Ok(bytes.clone());
|
||||
@@ -465,6 +496,9 @@ impl PartProofReader {
|
||||
bytes
|
||||
};
|
||||
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
if self.part.stripes <= 4 {
|
||||
put_verified_stripe_record(&self.part, stripe, record.clone());
|
||||
}
|
||||
if cached.records.len() == 2 {
|
||||
cached.records.pop_front();
|
||||
}
|
||||
@@ -863,4 +897,41 @@ mod tests {
|
||||
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
|
||||
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let inline_bytes = prepared.inline_bytes().expect("inline bytes");
|
||||
let part = prepared.part;
|
||||
|
||||
let warm_reader = PartProofReader {
|
||||
part: part.clone(),
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: Some(inline_bytes),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
|
||||
assert!(
|
||||
get_verified_stripe_record(&part, 0).is_some(),
|
||||
"small verified stripe record should populate the verified stripe cache"
|
||||
);
|
||||
|
||||
let cached_reader = Arc::new(PartProofReader {
|
||||
part,
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
|
||||
assert!(
|
||||
cached_reader.verify(0, 0, shards[1]).await.is_err(),
|
||||
"cached record must still reject corrupt or mismatched shard payloads"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ use tokio_util::sync::CancellationToken;
|
||||
#[derive(Clone)]
|
||||
pub struct NamespaceLockFence {
|
||||
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
|
||||
local_probes: Arc<Vec<rustfs_lock::fast_lock::guard::FastLockGuardProbe>>,
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
|
||||
}
|
||||
@@ -44,6 +45,7 @@ impl NamespaceLockFence {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
local_probes: Arc::default(),
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
|
||||
}
|
||||
@@ -58,17 +60,28 @@ impl NamespaceLockFence {
|
||||
{
|
||||
return true;
|
||||
}
|
||||
self.signals.iter().any(|signal| signal.is_lost())
|
||||
self.signals.iter().any(|signal| signal.is_lost()) || self.local_probes.iter().any(|probe| probe.is_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn add_signal(&mut self, signal: Arc<rustfs_lock::distributed_lock::LockLostSignal>) {
|
||||
Arc::make_mut(&mut self.signals).push(signal);
|
||||
}
|
||||
|
||||
fn add_local_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
if let rustfs_lock::NamespaceLockGuard::Fast(guard) = guard
|
||||
&& let Some(probe) = guard.release_probe()
|
||||
{
|
||||
Arc::make_mut(&mut self.local_probes).push(probe);
|
||||
}
|
||||
}
|
||||
|
||||
fn extend(&mut self, other: &Self) {
|
||||
if !Arc::ptr_eq(&self.signals, &other.signals) {
|
||||
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
|
||||
}
|
||||
if !Arc::ptr_eq(&self.local_probes, &other.local_probes) {
|
||||
Arc::make_mut(&mut self.local_probes).extend(other.local_probes.iter().cloned());
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
|
||||
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
|
||||
@@ -106,6 +119,7 @@ impl NamespaceLockSignalTestFence {
|
||||
) -> Self {
|
||||
let fence = NamespaceLockFence {
|
||||
signals: Arc::default(),
|
||||
local_probes: Arc::default(),
|
||||
forced_lost: Arc::new(vec![loss_handle]),
|
||||
};
|
||||
let signal_key = Arc::as_ptr(signal) as usize;
|
||||
@@ -160,6 +174,20 @@ pub struct ObjectLockConfigSnapshot {
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
|
||||
self._lifecycle_guard.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
|
||||
self.bucket.as_deref() == Some(bucket)
|
||||
&& incarnation.is_some()
|
||||
&& self.bucket_incarnation_id == incarnation
|
||||
&& self
|
||||
._lifecycle_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
|
||||
Self {
|
||||
store_id: None,
|
||||
@@ -208,6 +236,7 @@ impl ObjectLockConfigSnapshot {
|
||||
if let Some(signal) = lifecycle_guard.lock_lost_signal() {
|
||||
lifecycle_fence.add_signal(signal);
|
||||
}
|
||||
lifecycle_fence.add_local_guard(&lifecycle_guard);
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
bucket: Some(bucket.to_string()),
|
||||
@@ -1353,6 +1382,13 @@ impl ObjectOptions {
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
self.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
if let rustfs_lock::NamespaceLockGuard::Fast(local) = guard
|
||||
&& !local.is_disabled()
|
||||
{
|
||||
self.namespace_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
.add_local_guard(guard);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
@@ -1360,6 +1396,7 @@ impl ObjectOptions {
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
fence.add_signal(signal);
|
||||
}
|
||||
fence.add_local_guard(guard);
|
||||
}
|
||||
|
||||
pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) {
|
||||
@@ -1543,6 +1580,8 @@ pub struct ObjectInfo {
|
||||
/// published by the same upload. Callers must not repeat quota, replication,
|
||||
/// lifecycle, or object-created side effects for that response.
|
||||
pub multipart_completion_replayed: bool,
|
||||
/// Previous current version's size observed during `rename_data` commit.
|
||||
pub backfilled_old_current_size: Option<crate::disk::OldCurrentSize>,
|
||||
}
|
||||
|
||||
impl Default for ObjectInfo {
|
||||
@@ -1585,6 +1624,7 @@ impl Default for ObjectInfo {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1628,6 +1668,7 @@ impl Clone for ObjectInfo {
|
||||
replication_decision: self.replication_decision.clone(),
|
||||
checksum: self.checksum.clone(),
|
||||
multipart_completion_replayed: self.multipart_completion_replayed,
|
||||
backfilled_old_current_size: self.backfilled_old_current_size,
|
||||
expires: self.expires,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
@@ -71,6 +71,9 @@ const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
|
||||
|
||||
pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
|
||||
|
||||
// Bound abandoned speculative RPCs independently for each storage instance.
|
||||
pub(crate) const SPECULATIVE_RESERVATION_LIMIT: usize = 64;
|
||||
|
||||
/// A lease is deliberately short-lived. The coordinator treats expiry as a
|
||||
/// failed publication rather than silently continuing with a peer that may
|
||||
/// have started movement after the lease was abandoned.
|
||||
@@ -101,6 +104,12 @@ pub(crate) struct NamespaceCommitGuard {
|
||||
counted: bool,
|
||||
}
|
||||
|
||||
impl NamespaceCommitGuard {
|
||||
pub(crate) fn covers_instance(&self, ctx: &Arc<InstanceContext>) -> bool {
|
||||
Arc::ptr_eq(&self.ctx, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NamespaceCommitGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.counted {
|
||||
@@ -332,6 +341,7 @@ pub struct InstanceContext {
|
||||
scanner_publication_state: AtomicU8,
|
||||
namespace_commits: AtomicU64,
|
||||
namespace_commit_generation: AtomicU64,
|
||||
speculative_reservations: Arc<Semaphore>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
delete_recovery_hints: DeleteRecoveryHints,
|
||||
@@ -392,6 +402,7 @@ impl InstanceContext {
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
namespace_commits: AtomicU64::new(0),
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
speculative_reservations: Arc::new(Semaphore::new(SPECULATIVE_RESERVATION_LIMIT)),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
delete_recovery_hints: DeleteRecoveryHints::default(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -413,6 +424,11 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
/// Skip speculative work when its detached owners already fill the instance budget.
|
||||
pub(crate) fn try_acquire_speculative_reservation(&self) -> Option<OwnedSemaphorePermit> {
|
||||
Arc::clone(&self.speculative_reservations).try_acquire_owned().ok()
|
||||
}
|
||||
|
||||
/// Run a multi-step mutation on its own task and wait for its output.
|
||||
///
|
||||
/// Dropping the returned future leaves the task running to completion, so
|
||||
@@ -941,6 +957,32 @@ impl InstanceContext {
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
|
||||
///
|
||||
/// Called from `ECStore::drop`. When test fixtures share the bootstrap
|
||||
/// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
|
||||
/// reference cycle; once the store drops the weak is dangling and any
|
||||
/// `object_store()` call panics. Clearing the slot here lets the next
|
||||
/// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
|
||||
pub fn clear_bucket_metadata_sys_if_stale(&self) {
|
||||
let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
|
||||
if let Some(sys) = guard.as_ref() {
|
||||
// `read()` may block briefly; the async RwLock is acceptable here
|
||||
// because Drop is sync and the lock is held only long enough to
|
||||
// check the weak reference.
|
||||
let is_stale = {
|
||||
let s = sys.try_read();
|
||||
match s {
|
||||
Ok(s) => s.is_store_stale(),
|
||||
Err(_) => true, // contended: treat as stale-safe to clear
|
||||
}
|
||||
};
|
||||
if is_stale {
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Set this instance's background-services cancellation token (once).
|
||||
pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
|
||||
self.background_cancel_token.set(token)
|
||||
@@ -1117,6 +1159,26 @@ mod tests {
|
||||
assert!(ledger.admit(&bucket, &format!("{object_prefix}1024"), (None, false), false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speculative_reservation_budget_is_bounded_and_instance_local() {
|
||||
let first = InstanceContext::new();
|
||||
let other = InstanceContext::new();
|
||||
let mut permits = (0..super::SPECULATIVE_RESERVATION_LIMIT)
|
||||
.map(|_| first.try_acquire_speculative_reservation().expect("initial permit"))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(first.try_acquire_speculative_reservation().is_none());
|
||||
assert!(
|
||||
other.try_acquire_speculative_reservation().is_some(),
|
||||
"budgets belong to their own instance"
|
||||
);
|
||||
drop(permits.pop());
|
||||
let replacement = first
|
||||
.try_acquire_speculative_reservation()
|
||||
.expect("dropping an owner frees its slot");
|
||||
assert!(first.try_acquire_speculative_reservation().is_none());
|
||||
drop(replacement);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
|
||||
let first = Arc::new(InstanceContext::new());
|
||||
@@ -1125,6 +1187,8 @@ mod tests {
|
||||
other.set_scanner_publication_state(false);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
let one = first.begin_namespace_commit();
|
||||
assert!(one.covers_instance(&first));
|
||||
assert!(!one.covers_instance(&other));
|
||||
let shared_owner = Arc::clone(&one);
|
||||
let two = first.begin_namespace_commit();
|
||||
assert!(first.namespace_commits_pending());
|
||||
|
||||
@@ -2538,6 +2538,8 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
|
||||
pub(in crate::set_disk) tail_has_owned_bucket_fences: bool,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
@@ -3468,6 +3470,8 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
|
||||
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
@@ -3483,6 +3487,8 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: None,
|
||||
multipart_proof_authority: None,
|
||||
multipart_commit_owner: None,
|
||||
reservation_ctx: None,
|
||||
prepared_reservations: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3519,6 +3525,415 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_reservation_context(
|
||||
mut self,
|
||||
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
) -> Self {
|
||||
self.reservation_ctx = ctx;
|
||||
self.prepared_reservations = reservations;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MetadataRenewalTarget {
|
||||
disk: DiskStore,
|
||||
token: Uuid,
|
||||
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
|
||||
}
|
||||
|
||||
struct MetadataRenewalGroup {
|
||||
targets: Arc<parking_lot::Mutex<HashMap<usize, MetadataRenewalTarget>>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Drop for MetadataRenewalGroup {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameMetadataReservation {
|
||||
disk: DiskStore,
|
||||
token: Option<Uuid>,
|
||||
// Remote reservation protocol v2 and the local owned path both retain
|
||||
// bucket publication and lifecycle guards until the physical write drains.
|
||||
owns_bucket_fence: bool,
|
||||
short: bool,
|
||||
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
|
||||
renewal: Option<(Arc<MetadataRenewalGroup>, usize)>,
|
||||
}
|
||||
|
||||
impl RenameMetadataReservation {
|
||||
fn stop_renewal(&mut self) {
|
||||
if let Some((group, index)) = self.renewal.take() {
|
||||
let mut targets = group.targets.lock();
|
||||
targets.remove(&index);
|
||||
if targets.is_empty() {
|
||||
group.task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cancel(&mut self) -> disk::error::Result<()> {
|
||||
self.stop_renewal();
|
||||
let Some(token) = self.token else { return Ok(()) };
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
|
||||
Ok(Ok(true)) => {
|
||||
self.token = None;
|
||||
Ok(())
|
||||
}
|
||||
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameMetadataReservation {
|
||||
fn drop(&mut self) {
|
||||
self.stop_renewal();
|
||||
let Some(token) = self.token.take() else { return };
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let disk = self.disk.clone();
|
||||
runtime.spawn(async move {
|
||||
let _ = disk.cancel_object_metadata_reservation(token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum RenameReservationMode {
|
||||
Ownerless,
|
||||
Owned,
|
||||
ShortOwned,
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
owns_bucket_fence: bool,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
prepare_rename_metadata_reservations_until_abandoned(
|
||||
disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
ctx,
|
||||
if owns_bucket_fence {
|
||||
RenameReservationMode::Owned
|
||||
} else {
|
||||
RenameReservationMode::Ownerless
|
||||
},
|
||||
std::future::pending(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations_until_abandoned(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
mode: RenameReservationMode,
|
||||
abandoned: impl std::future::Future<Output = ()>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
let owns_bucket_fence = !matches!(mode, RenameReservationMode::Ownerless);
|
||||
let short = matches!(mode, RenameReservationMode::ShortOwned);
|
||||
let mut abandoned = Box::pin(abandoned);
|
||||
let discarded = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let mut pending: FuturesUnordered<_> = disks
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(_index, disk)| {
|
||||
let discarded = Arc::clone(&discarded);
|
||||
async move {
|
||||
let result: disk::error::Result<Option<RenameMetadataReservation>> = async {
|
||||
if discarded.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(disk) = disk else { return Ok(None) };
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_BEFORE).await;
|
||||
if disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, disk, None, false).await?;
|
||||
}
|
||||
if discarded.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return Ok(None);
|
||||
}
|
||||
let prepared_at = tokio::time::Instant::now();
|
||||
let token = if disk.is_local() && owns_bucket_fence {
|
||||
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or(DiskError::DiskOngoingReq)?;
|
||||
if short {
|
||||
store
|
||||
.reserve_local_object_metadata_short(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_millis(50),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
} else if short {
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
remote
|
||||
.reserve_object_metadata_short(bucket, object, incarnation, Duration::from_millis(50))
|
||||
.await?
|
||||
} else {
|
||||
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
|
||||
.await?
|
||||
};
|
||||
// No await separates receipt from installing the cancellation owner.
|
||||
let reservation = RenameMetadataReservation {
|
||||
disk: disk.clone(),
|
||||
token: Some(token),
|
||||
owns_bucket_fence: !disk.is_local() || owns_bucket_fence,
|
||||
short,
|
||||
renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)),
|
||||
renewal: None,
|
||||
};
|
||||
if short {
|
||||
// Install the cancellation owner before extending the short token.
|
||||
// A dropped renew response still leaves us the token to cancel.
|
||||
disk.renew_object_metadata(bucket, object, token, Duration::from_millis(50))
|
||||
.await?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(object, _index, rename_fanout_barrier::PHASE_RESERVATION_AFTER).await;
|
||||
Ok(Some(reservation))
|
||||
}
|
||||
.await;
|
||||
(_index, result)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut reservations: Vec<Option<RenameMetadataReservation>> = (0..disks.len()).map(|_| None).collect();
|
||||
let mut discard = false;
|
||||
while !pending.is_empty() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut abandoned, if !discard => {
|
||||
discard = true;
|
||||
discarded.store(true, std::sync::atomic::Ordering::Release);
|
||||
reservations.clear();
|
||||
}
|
||||
result = pending.next() => {
|
||||
let Some((index, result)) = result else { break };
|
||||
if discard {
|
||||
drop(result);
|
||||
} else {
|
||||
match result {
|
||||
Ok(reservation) => reservations[index] = reservation,
|
||||
Err(_) => {
|
||||
discard = true;
|
||||
discarded.store(true, std::sync::atomic::Ordering::Release);
|
||||
reservations.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if discard {
|
||||
// Drain all in-flight preparations so late tokens get a cancellation owner.
|
||||
return Ok(None);
|
||||
}
|
||||
let targets = Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
for (index, reservation) in reservations.iter().enumerate() {
|
||||
if let Some(reservation) = reservation {
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
targets.lock().insert(
|
||||
index,
|
||||
MetadataRenewalTarget {
|
||||
disk: reservation.disk.clone(),
|
||||
token,
|
||||
renewed_at: Arc::clone(&reservation.renewed_at),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
if !targets.lock().is_empty() {
|
||||
let renewal_targets = Arc::clone(&targets);
|
||||
let renewal_ctx = Arc::clone(ctx);
|
||||
let renewal_bucket = bucket.to_owned();
|
||||
let renewal_object = object.to_owned();
|
||||
// Short leases remain short even after renewal, so a lost renewal and
|
||||
// cancellation cannot strand a bucket fence for the legacy lease time.
|
||||
let task = tokio::spawn(async move {
|
||||
let interval = if short {
|
||||
Duration::from_millis(500)
|
||||
} else {
|
||||
Duration::from_secs(20)
|
||||
};
|
||||
let retry = if short {
|
||||
Duration::from_millis(250)
|
||||
} else {
|
||||
Duration::from_secs(5)
|
||||
};
|
||||
let lifetime = if short {
|
||||
Duration::from_secs(2)
|
||||
} else {
|
||||
Duration::from_secs(60)
|
||||
};
|
||||
let mut delay = interval;
|
||||
loop {
|
||||
tokio::time::sleep(delay).await;
|
||||
let current: Vec<_> = renewal_targets.lock().values().cloned().collect();
|
||||
if current.is_empty() {
|
||||
break;
|
||||
}
|
||||
// Once any acknowledgment is too old, the complete fanout can
|
||||
// no longer be safely activated. Stop retrying stale peers;
|
||||
// their receiver-side tokens still expire on their own.
|
||||
if current.iter().any(|target| target.renewed_at.lock().elapsed() >= lifetime) {
|
||||
break;
|
||||
}
|
||||
let results = join_all(current.into_iter().map(|target| {
|
||||
let ctx = Arc::clone(&renewal_ctx);
|
||||
let bucket = &renewal_bucket;
|
||||
let object = &renewal_object;
|
||||
async move {
|
||||
let started = tokio::time::Instant::now();
|
||||
let admitted = !target.disk.is_local()
|
||||
|| crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok();
|
||||
if admitted
|
||||
&& target
|
||||
.disk
|
||||
.renew_object_metadata(bucket, object, target.token, retry)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
*target.renewed_at.lock() = started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
delay = if results.into_iter().all(|success| success) {
|
||||
interval
|
||||
} else {
|
||||
retry
|
||||
};
|
||||
}
|
||||
});
|
||||
let group = Arc::new(MetadataRenewalGroup { targets, task });
|
||||
for (index, reservation) in reservations.iter_mut().enumerate() {
|
||||
if let Some(reservation) = reservation {
|
||||
reservation.renewal = Some((Arc::clone(&group), index));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
pub(crate) fn align_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<Vec<Option<RenameMetadataReservation>>> {
|
||||
if reservations.len() != disks.len() {
|
||||
return None;
|
||||
}
|
||||
let mut aligned = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
aligned.push(None);
|
||||
continue;
|
||||
};
|
||||
let index = reservations
|
||||
.iter()
|
||||
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
|
||||
aligned.push(reservations[index].take());
|
||||
}
|
||||
// Disks lost while staging need no activation; dropping their still-inert
|
||||
// reservations schedules cancellation without blocking the commit.
|
||||
Some(aligned)
|
||||
}
|
||||
|
||||
async fn activate_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
// Short tokens continue renewing during activation; an activated token
|
||||
// can otherwise expire while another disk's activation is still pending.
|
||||
// Long tokens retain their existing non-renewable queued behavior.
|
||||
for reservation in reservations.iter_mut().flatten() {
|
||||
if !reservation.short {
|
||||
reservation.stop_renewal();
|
||||
}
|
||||
}
|
||||
if reservations.iter().flatten().any(|reservation| {
|
||||
let max_age = if reservation.short {
|
||||
Duration::from_secs(1)
|
||||
} else {
|
||||
Duration::from_secs(50)
|
||||
};
|
||||
reservation.renewed_at.lock().elapsed() >= max_age
|
||||
}) {
|
||||
return Ok(None);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_reservations = &reservations;
|
||||
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
|
||||
if reservation.disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
|
||||
}
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
let result = reservation
|
||||
.disk
|
||||
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
if result.is_ok() {
|
||||
let index = test_reservations
|
||||
.iter()
|
||||
.position(|candidate| {
|
||||
candidate
|
||||
.as_ref()
|
||||
.is_some_and(|candidate| std::ptr::eq(candidate, reservation))
|
||||
})
|
||||
.expect("activated reservation belongs to the fanout");
|
||||
rename_fanout_barrier::checkpoint(object, index, rename_fanout_barrier::PHASE_RESERVATION_ACTIVATED).await;
|
||||
}
|
||||
result
|
||||
}))
|
||||
.await;
|
||||
for reservation in reservations.iter_mut().flatten() {
|
||||
reservation.stop_renewal();
|
||||
}
|
||||
if activated.into_iter().any(|result| result.is_err()) {
|
||||
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
|
||||
if cancelled.into_iter().any(|result| result.is_err()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -3530,7 +3945,7 @@ type RenameDataLegacyTuple = (
|
||||
Option<OldCurrentSize>,
|
||||
);
|
||||
|
||||
fn put_rename_early_ack_enabled() -> bool {
|
||||
pub(crate) fn put_rename_early_ack_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
|
||||
}
|
||||
|
||||
@@ -3896,6 +4311,8 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
tail_has_owned_bucket_fences: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3992,6 +4409,8 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority: _,
|
||||
multipart_commit_owner: _,
|
||||
reservation_ctx,
|
||||
prepared_reservations,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4007,6 +4426,21 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
|
||||
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
|
||||
(_, _) => None,
|
||||
};
|
||||
let tail_has_metadata_reservations = reservations.is_some();
|
||||
let tail_has_owned_bucket_fences = reservations.as_ref().is_some_and(|reservations| {
|
||||
reservations.len() == disks.len()
|
||||
&& disks.iter().any(Option::is_some)
|
||||
&& disks.iter().zip(reservations).all(|(disk, reservation)| match disk {
|
||||
Some(disk) => reservation
|
||||
.as_ref()
|
||||
.is_some_and(|owner| owner.owns_bucket_fence && Arc::ptr_eq(&owner.disk, disk)),
|
||||
None => true,
|
||||
})
|
||||
});
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -4039,6 +4473,11 @@ impl SetDisks {
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let reservation = reservations
|
||||
.as_mut()
|
||||
.and_then(|entries| entries.get_mut(i))
|
||||
.and_then(Option::take);
|
||||
let reservation_ctx = reservation_ctx.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4049,6 +4488,17 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_admission = if reservation.is_some() && disk.is_local() {
|
||||
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
|
||||
// The fanout owner already covers this instance through
|
||||
// physical completion; only add a disk owner when it does not.
|
||||
let covered = namespace_commit_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.covers_instance(ctx));
|
||||
crate::store::admit_local_disk(ctx, &disk, None, !covered).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
@@ -4074,6 +4524,11 @@ impl SetDisks {
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
|
||||
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
|
||||
} else {
|
||||
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
|
||||
};
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4083,9 +4538,8 @@ impl SetDisks {
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
|
||||
namespace_owner,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4363,6 +4817,8 @@ impl SetDisks {
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
|
||||
commit.tail_has_owned_bucket_fences = tail_has_owned_bucket_fences;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
@@ -4432,6 +4888,8 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority,
|
||||
multipart_commit_owner,
|
||||
reservation_ctx: _,
|
||||
prepared_reservations: _,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4901,6 +5359,8 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
tail_has_owned_bucket_fences: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5340,6 +5800,14 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
|
||||
@@ -5539,6 +6007,14 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
@@ -5598,10 +6074,11 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
let part = ObjectPartInfo::unmarshal(&meta)?;
|
||||
let integrity = part.integrity.map(Arc::new);
|
||||
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
@@ -5623,7 +6100,9 @@ impl SetDisks {
|
||||
if let Err(error) = prepared {
|
||||
return Some(Err(error));
|
||||
}
|
||||
if let Some(integrity) = integrity {
|
||||
if let Some(integrity) = integrity
|
||||
&& !disk.is_local()
|
||||
{
|
||||
let Some((directory, _)) = dst_object.rsplit_once('/') else {
|
||||
return Some(Err(DiskError::FileCorrupt));
|
||||
};
|
||||
@@ -5740,7 +6219,36 @@ impl SetDisks {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
let all_prepared_succeeded = errs
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
|
||||
let committed = if all_prepared_succeeded {
|
||||
let settles = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Commit)
|
||||
.await
|
||||
}
|
||||
});
|
||||
let settle_results = join_all(settles).await;
|
||||
if settle_results.iter().filter(|res| res.is_ok()).count() >= write_quorum
|
||||
&& settle_results
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(res, disk)| disk.is_none() || res.is_ok())
|
||||
{
|
||||
true
|
||||
} else {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?
|
||||
}
|
||||
} else {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?
|
||||
};
|
||||
if !committed {
|
||||
let err = DiskError::ErasureWriteQuorum;
|
||||
if let Some(context) = quorum_context {
|
||||
@@ -6737,9 +7245,7 @@ impl SetDisks {
|
||||
object,
|
||||
);
|
||||
}
|
||||
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
||||
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
||||
if should_prevent_write(&oi, if_none_match, if_match)
|
||||
if should_prevent_write(&oi, http_preconditions.if_none_match_value(), http_preconditions.if_match_value())
|
||||
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
|
||||
{
|
||||
return Some(StorageError::PreconditionFailed);
|
||||
@@ -7312,6 +7818,9 @@ pub(crate) mod rename_fanout_barrier {
|
||||
MetadataSlowtailFault,
|
||||
NonInlineHedgeTimer,
|
||||
LateMetadataRefresh,
|
||||
ReservationBefore,
|
||||
ReservationAfter,
|
||||
ReservationActivated,
|
||||
}
|
||||
|
||||
impl From<&'static str> for Phase {
|
||||
@@ -7334,6 +7843,9 @@ pub(crate) mod rename_fanout_barrier {
|
||||
pub const PHASE_ROLLBACK: Phase = Phase::Rollback;
|
||||
pub const PHASE_READ_VERSION: Phase = Phase::ReadVersion;
|
||||
pub const PHASE_METADATA_SLOWTAIL_FAULT: Phase = Phase::MetadataSlowtailFault;
|
||||
pub const PHASE_RESERVATION_BEFORE: Phase = Phase::ReservationBefore;
|
||||
pub const PHASE_RESERVATION_AFTER: Phase = Phase::ReservationAfter;
|
||||
pub const PHASE_RESERVATION_ACTIVATED: Phase = Phase::ReservationActivated;
|
||||
|
||||
/// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk.
|
||||
pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer;
|
||||
@@ -7408,6 +7920,18 @@ pub(crate) mod rename_fanout_barrier {
|
||||
}
|
||||
|
||||
impl BarrierHandle {
|
||||
/// Keep the current task parked without intercepting later tasks.
|
||||
pub fn disarm(&self) {
|
||||
let mut reg = lock();
|
||||
if reg
|
||||
.armed
|
||||
.get(&self.key)
|
||||
.is_some_and(|armed| Arc::ptr_eq(&armed.release, &self.release))
|
||||
{
|
||||
reg.armed.remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Await until the armed fan-out task has parked at the checkpoint. Uses a
|
||||
/// stored-permit `Notify`, so this is race-free regardless of whether the
|
||||
/// task reaches the checkpoint before or after this call — no sleeps.
|
||||
@@ -7573,6 +8097,45 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() {
|
||||
let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await;
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(endpoint.clone(), Some(disk.clone()));
|
||||
let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false)
|
||||
.await
|
||||
.expect("prepare")
|
||||
.expect("complete fanout");
|
||||
let group = Arc::clone(
|
||||
&reservations[0]
|
||||
.as_ref()
|
||||
.expect("reservation")
|
||||
.renewal
|
||||
.as_ref()
|
||||
.expect("renewal")
|
||||
.0,
|
||||
);
|
||||
ctx.local_disk_map().write().await.remove(&endpoint);
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(21)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(41)).await;
|
||||
for _ in 0..10 {
|
||||
if group.task.is_finished() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::resume();
|
||||
assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts");
|
||||
drop(reservations);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() {
|
||||
use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint};
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for parts in versions.into_values() {
|
||||
for (version_id, parts) in versions {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,15 +173,25 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
|
||||
@@ -80,7 +80,7 @@ use crate::erasure::codec::bridge::{
|
||||
use crate::erasure::coding;
|
||||
use crate::error::{Error, Result, is_err_version_not_found};
|
||||
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer, create_bitrot_writer_with_orig_volume};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::get_object_body_cache_hook;
|
||||
use crate::object_api::object_api_utils::get_raw_etag;
|
||||
@@ -120,7 +120,6 @@ use bytesize::ByteSize;
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use regex::Regex;
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_filemeta::{
|
||||
@@ -411,6 +410,10 @@ impl ObjectLockDiagGuard {
|
||||
self
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
self.guard.downgrade_committed_tail().await
|
||||
}
|
||||
|
||||
/// Whether the underlying namespace lock's heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
|
||||
/// point on this so a stale lock holder does not race a double-write.
|
||||
@@ -903,6 +906,10 @@ mod core;
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::{
|
||||
RenameMetadataReservation, RenameReservationMode, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
|
||||
prepare_rename_metadata_reservations_until_abandoned, put_rename_early_ack_enabled,
|
||||
};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
@@ -929,6 +936,8 @@ pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::pause_put_object_commit;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::{DeleteCleanupBarrier, DeleteObjectCommitBarrier};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
@@ -8036,16 +8045,29 @@ fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Opti
|
||||
Some(&part_numbers[index..])
|
||||
}
|
||||
|
||||
pub fn canonicalize_etag(etag: &str) -> String {
|
||||
let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
|
||||
re.replace_all(etag, "$1").to_string()
|
||||
fn canonicalized_etag(etag: &str) -> Cow<'_, str> {
|
||||
// Preserve the legacy suffix-matching behavior for embedded quotes.
|
||||
let trimmed = etag.trim_end_matches('"');
|
||||
let Some(last_quote) = trimmed.rfind('"') else {
|
||||
return Cow::Borrowed(trimmed);
|
||||
};
|
||||
let prefix = trimmed[..last_quote].trim_end_matches('"');
|
||||
let suffix = &trimmed[last_quote + 1..];
|
||||
if prefix.is_empty() {
|
||||
Cow::Borrowed(suffix)
|
||||
} else {
|
||||
let mut result = String::with_capacity(prefix.len() + suffix.len());
|
||||
result.push_str(prefix);
|
||||
result.push_str(suffix);
|
||||
Cow::Owned(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
|
||||
if condition.trim() == "*" {
|
||||
return true;
|
||||
}
|
||||
canonicalize_etag(etag) == canonicalize_etag(condition)
|
||||
canonicalized_etag(etag) == canonicalized_etag(condition)
|
||||
}
|
||||
|
||||
fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
@@ -8060,15 +8082,12 @@ fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
.or_else(|| trimmed.strip_prefix("w/"))
|
||||
.unwrap_or(trimmed)
|
||||
}
|
||||
canonicalize_etag(strip_weak(etag)) == canonicalize_etag(strip_weak(condition))
|
||||
canonicalized_etag(strip_weak(etag)) == canonicalized_etag(strip_weak(condition))
|
||||
}
|
||||
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
|
||||
let if_none_match = if_none_match
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.as_deref().map(str::trim).filter(|condition| !condition.is_empty());
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<&str>, if_match: Option<&str>) -> bool {
|
||||
let if_none_match = if_none_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
|
||||
match &oi.etag {
|
||||
Some(etag) => {
|
||||
@@ -9029,6 +9048,13 @@ mod tests {
|
||||
"constructing an object lock must not clone each client Arc"
|
||||
);
|
||||
|
||||
// Check shared ownership before acquisition can spawn pending cleanup.
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
|
||||
let write_error = write_lock
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9043,11 +9069,6 @@ mod tests {
|
||||
),
|
||||
"the shared client representation must preserve the exact write quorum result: {write_error}"
|
||||
);
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
let read_guard = read_lock
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -13772,6 +13793,22 @@ mod tests {
|
||||
assert!(result2.iter().all(|d| d.is_none()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalized_etag_preserves_legacy_regex_semantics() {
|
||||
let regex = regex::Regex::new("\"*?([^\"]*?)\"*?$").expect("valid legacy ETag pattern");
|
||||
let alphabet = ['a', '"', '\n', 'é', '\0'];
|
||||
for length in 0..=6_u32 {
|
||||
for mut number in 0..alphabet.len().pow(length) {
|
||||
let mut input = String::new();
|
||||
for _ in 0..length {
|
||||
input.push(alphabet[number % alphabet.len()]);
|
||||
number /= alphabet.len();
|
||||
}
|
||||
assert_eq!(canonicalized_etag(&input), regex.replace_all(&input, "$1"), "input: {input:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etag_matches() {
|
||||
assert!(e_tag_matches("abc", "abc"));
|
||||
@@ -14347,61 +14384,61 @@ mod tests {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("abc".to_string());
|
||||
let if_none_match = Some("abc");
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("def".to_string());
|
||||
let if_match = Some("def");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("*".to_string());
|
||||
let if_match = Some("*");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("*".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("*");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: None,
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some(String::new());
|
||||
let if_match = Some(" ".to_string());
|
||||
let if_none_match = Some("");
|
||||
let if_match = Some(" ");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\"".to_string()), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\"".to_string()), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"".to_string())));
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\""), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\""), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -34,6 +34,7 @@ mod heal_lock_wait_tests;
|
||||
#[derive(Clone, Copy)]
|
||||
struct ExplicitVersionHeal<'a> {
|
||||
opts: &'a HealOpts,
|
||||
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
|
||||
allow_regeneration: bool,
|
||||
preserve_unrepairable: bool,
|
||||
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
|
||||
@@ -41,6 +42,58 @@ struct ExplicitVersionHeal<'a> {
|
||||
retirement: Option<&'a MarkerRetirementContext<'a>>,
|
||||
}
|
||||
|
||||
async fn restore_heal_exclusive_guard(
|
||||
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> disk::error::Result<()> {
|
||||
tokio::time::timeout(get_lock_acquire_timeout(), async {
|
||||
loop {
|
||||
{
|
||||
let mut guard = guard.lock().await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// A writer may have waited on a peer that upgraded while
|
||||
// another peer was still waiting for its readers to drain.
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
return if matches!(waiting, Ok(Some(false))) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::Timeout)
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let waiting =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
|
||||
drop(tokio::spawn(async move {
|
||||
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}));
|
||||
}
|
||||
|
||||
// Keep the large heal state machine and its construction temporary out of
|
||||
// each dispatch caller's poll frame. Ordinary object writes do not use this
|
||||
// heal-only allocation boundary.
|
||||
@@ -844,6 +897,7 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: None,
|
||||
@@ -994,6 +1048,7 @@ impl SetDisks {
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
let ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard,
|
||||
allow_regeneration: allow_explicit_version_regen,
|
||||
preserve_unrepairable,
|
||||
commit_fence,
|
||||
@@ -1701,7 +1756,29 @@ impl SetDisks {
|
||||
let dst_data_dir = data_dir;
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = async {
|
||||
let mut shared_staging = false;
|
||||
if let Some(guard) = staging_guard {
|
||||
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.supports_heal_lock_transition().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if supported {
|
||||
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.downgrade_committed_tail().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if converted {
|
||||
shared_staging = true;
|
||||
} else {
|
||||
// Some peers may have converted. Abort without committing;
|
||||
// dropping the guard releases each retained lease.
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stage = async {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
|
||||
|
||||
@@ -1926,17 +2003,43 @@ impl SetDisks {
|
||||
|
||||
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
|
||||
Ok(None)
|
||||
}
|
||||
.await;
|
||||
};
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
|
||||
if let Some(guard) = staging_guard {
|
||||
tokio::select! {
|
||||
result = stage => result,
|
||||
_ = wait_for_heal_writer(guard) => {
|
||||
// The UUID prefix is private to this attempt. Cleanup must not
|
||||
// retain the namespace lock while a foreground writer waits;
|
||||
// the stale-tmp sweeper also covers a cancelled disk operation.
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
};
|
||||
match stage_result {
|
||||
Ok(None) => {}
|
||||
failure => {
|
||||
// Release readers and writers before removing all parts
|
||||
// staged under this heal's exclusively owned prefix.
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
if shared_staging {
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
} else {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}
|
||||
return failure.map(|error| (result, error));
|
||||
}
|
||||
}
|
||||
if shared_staging
|
||||
&& let Some(guard) = staging_guard
|
||||
&& let Err(error) = restore_heal_exclusive_guard(guard).await
|
||||
{
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
|
||||
if _read_repair_read_lock_guard
|
||||
@@ -3616,6 +3719,7 @@ impl SetDisks {
|
||||
&version,
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(can_commit),
|
||||
@@ -3676,6 +3780,7 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &read_opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -3702,7 +3807,7 @@ impl SetDisks {
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
|
||||
Some(Arc::new(
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
measure_heal_object_lock(
|
||||
"heal_object",
|
||||
HealObjectLockKind::Write,
|
||||
@@ -3714,7 +3819,7 @@ impl SetDisks {
|
||||
.narrow_to_disk()
|
||||
.unwrap_or_else(DiskError::other)
|
||||
})?,
|
||||
))
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -3732,10 +3837,18 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
|
||||
guard.lock().await.lock_lost_signal()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
// Recovery cleanup can outlive its future. Never permit reads until
|
||||
// every physical operation holding its recovery owner has finished.
|
||||
let recovery_tail = Arc::new(());
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3762,6 +3875,7 @@ impl SetDisks {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
Arc::clone(&recovery_tail),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -3885,6 +3999,9 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -3909,6 +4026,9 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -6047,6 +6167,7 @@ mod heal_result_report_tests {
|
||||
"",
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -10637,6 +10758,369 @@ mod heal_result_report_tests {
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-storage-entry";
|
||||
let object = "object.bin";
|
||||
let expected = vec![0x37; 1024 * 1024];
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external object data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing).await.expect("remove shard");
|
||||
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
let result = heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair: false,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, absence)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("read-repair reached shard reconstruction");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
set.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("background heal reconstruction should allow concurrent readers")
|
||||
.expect("open object while reconstruction is paused");
|
||||
let mut during_heal = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
|
||||
.await
|
||||
.expect("read completes while reconstruction is paused")
|
||||
.expect("read object during reconstruction");
|
||||
assert_eq!(during_heal, expected);
|
||||
drop(reader);
|
||||
release.notify_one();
|
||||
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("repair finishes")
|
||||
.expect("repair task joins");
|
||||
let (_, error) = result.expect("repair operation succeeds");
|
||||
assert!(error.is_none(), "repair failed: {error:?}");
|
||||
assert!(absence.is_none());
|
||||
assert!(missing.exists(), "missing shard was not restored");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-long-put";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let put = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes after releasing reconstruction")
|
||||
.expect("heal task joins");
|
||||
put.expect("foreground PUT finishes during reconstruction")
|
||||
.expect("foreground PUT must not time out");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
drop(_pause);
|
||||
let mut absence = None;
|
||||
let (_, retry_error) = set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a later heal must retry the current object");
|
||||
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read foreground write after heal");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-version-delete";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let original = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original version");
|
||||
let version = original.version_id.expect("version id").to_string();
|
||||
let survivor = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x52; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write surviving version")
|
||||
.version_id
|
||||
.expect("surviving version id");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("read original metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal_version = version.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
&heal_version,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut None,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let delete = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes")
|
||||
.expect("heal task joins");
|
||||
delete
|
||||
.expect("foreground delete finishes")
|
||||
.expect("foreground delete succeeds");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
let result = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
|
||||
"stale heal must not resurrect the deleted version: {result:?}"
|
||||
);
|
||||
let mut survivor = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(survivor.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version remains readable");
|
||||
let mut body = Vec::new();
|
||||
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
|
||||
assert_eq!(body, vec![0x52; 1024]);
|
||||
for disk in &disks {
|
||||
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
|
||||
assert!(
|
||||
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
|
||||
"deleted version must remain absent on each disk: {physical:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -41,8 +41,8 @@ use super::super::{
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
|
||||
create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
|
||||
get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
|
||||
get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
|
||||
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
@@ -124,9 +124,10 @@ async fn create_part_writers(
|
||||
let Some(disk) = disk else {
|
||||
return (None, Some(DiskError::DiskNotFound));
|
||||
};
|
||||
match create_bitrot_writer(
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
false,
|
||||
Some(disk),
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
path,
|
||||
length,
|
||||
@@ -977,7 +978,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() {
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1002,6 +1003,13 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.into_iter()
|
||||
.map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
|
||||
.filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.map(|sha_dir| {
|
||||
if root_prefix.is_empty() {
|
||||
sha_dir
|
||||
} else {
|
||||
format!("{root_prefix}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let listed_upload_dirs = stream::iter(sha_dirs)
|
||||
.map(|sha_dir| {
|
||||
@@ -1124,6 +1132,7 @@ impl SetDisks {
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -1165,7 +1174,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
|
||||
let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
|
||||
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
|
||||
}
|
||||
|
||||
@@ -1187,7 +1197,7 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
@@ -1200,8 +1210,9 @@ impl SetDisks {
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
let (_, candidate_paths, _, has_minority_candidate) = self
|
||||
.discover_multipart_upload_paths(bucket, object, &expected_parent, false)
|
||||
.await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::DecommissionCapacityBlocked {
|
||||
message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
|
||||
@@ -1750,6 +1761,54 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
// Every disk must confirm absence in both namespaces before cleanup can
|
||||
// remove catalog protection from the bucket.
|
||||
for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let (disks, paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for upload_path in paths {
|
||||
let (sha_dir, _) = upload_path
|
||||
.rsplit_once('/')
|
||||
.filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
|
||||
.ok_or(DiskError::CorruptedFormat)?;
|
||||
let (metadata, errors) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
|
||||
.await?;
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
|
||||
let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
|
||||
let (Some(owner_bucket), Some(owner_object)) = (
|
||||
file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) else {
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
};
|
||||
if owner_bucket.is_empty()
|
||||
|| owner_object.is_empty()
|
||||
|| Self::get_multipart_sha_dir(owner_bucket, owner_object)
|
||||
!= sha_dir
|
||||
.strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.unwrap_or(sha_dir)
|
||||
{
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
if owner_bucket == bucket {
|
||||
return Err(Error::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1761,7 +1820,8 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -2372,7 +2432,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let multipart_part_size = data.size();
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
|
||||
);
|
||||
|
||||
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
|
||||
@@ -2436,7 +2496,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
let part_integrity = if let Some(integrity) = integrity {
|
||||
Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
|
||||
Some(
|
||||
integrity
|
||||
.write(
|
||||
&mut shuffle_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else { None };
|
||||
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
@@ -3197,19 +3266,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let mut decommission_target_lock_covered = false;
|
||||
let mut decommission_capacity_guard = None;
|
||||
|
||||
if opts.http_preconditions.is_some() {
|
||||
if write_context.acquires_namespace() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_precondition", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -3221,7 +3277,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
self.pool_index,
|
||||
bucket,
|
||||
object,
|
||||
!write_context.acquires_namespace() || object_lock_guard.is_some(),
|
||||
!write_context.acquires_namespace(),
|
||||
false,
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?;
|
||||
@@ -3235,6 +3292,28 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
// Evaluate the precondition after taking the fixed decommission domain
|
||||
// and the target domain, so a staged upload on the retiring source
|
||||
// cannot publish while migration holds its source read anchor.
|
||||
if opts.http_preconditions.is_some() {
|
||||
// The store's earlier cross-pool check is advisory. A write in a
|
||||
// different pool can complete before this fixed-domain guard is
|
||||
// acquired, so recheck all pools while it excludes those writes.
|
||||
if decommission_object_lock_guard.is_some()
|
||||
&& !opts.data_movement
|
||||
&& !opts.replication_request
|
||||
&& !opts.no_lock
|
||||
&& !opts.skip_decommissioned
|
||||
&& !opts.skip_rebalancing
|
||||
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
|
||||
&& let Some(err) = store.check_write_precondition(bucket, object, opts, true).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
let upload_guard = self
|
||||
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
@@ -3471,10 +3550,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
bind_multipart_proof_snapshot(&disks, &self.multipart_proof_verification_context())?;
|
||||
|
||||
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let part_meta_paths = uploaded_parts
|
||||
.iter()
|
||||
.map(|v| format!("{part_path}part.{0}.meta", v.part_num))
|
||||
@@ -3482,6 +3557,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
|
||||
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let object_parts = Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
@@ -4389,6 +4468,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let needs_immediate_heal = rename_commit.needs_immediate_heal();
|
||||
let op_old_dir = rename_commit.data_dir;
|
||||
let cleanup_disks = rename_commit.cleanup_disks;
|
||||
let old_current_size = rename_commit.old_current_size;
|
||||
let committed_file_info = rename_commit.committed_file_info;
|
||||
|
||||
if needs_immediate_heal {
|
||||
@@ -4457,7 +4537,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
let can_skip_part_cleanup = !needs_upgrade
|
||||
&& commit_disks
|
||||
.iter()
|
||||
.all(|disk| disk.as_ref().is_some_and(|disk| disk.is_local()));
|
||||
if !can_skip_part_cleanup {
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
}
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
@@ -4503,7 +4589,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
drop(_upload_guard.take());
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
|
||||
let mut obj_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned);
|
||||
obj_info.backfilled_old_current_size = old_current_size;
|
||||
Ok(obj_info)
|
||||
};
|
||||
|
||||
let result = if detach_commit_owner {
|
||||
@@ -11194,6 +11282,163 @@ mod tests {
|
||||
assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-target";
|
||||
let other = "multipart-cleanup-other";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create target bucket");
|
||||
disk.make_volume(other).await.expect("create other bucket");
|
||||
}
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an empty upload namespace permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(other, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in other bucket");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an upload owned by another bucket permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in target bucket");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-offline";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks.disks.write().await[3] = None;
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing preserves its discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the offline disk must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-list-error";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
|
||||
let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
|
||||
tokio::fs::create_dir_all(&multipart_dir)
|
||||
.await
|
||||
.expect("create multipart staging directory");
|
||||
tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
|
||||
.await
|
||||
.expect("replace one upload parent directory with a file");
|
||||
multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
|
||||
.await
|
||||
.expect_err("the affected disk must fail directory enumeration");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing still accepts the remaining discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
|
||||
.await
|
||||
.expect("remove the invalid directory entry");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after repair");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_data_movement_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-data-movement";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
|
||||
);
|
||||
set_disks
|
||||
.new_multipart_upload(
|
||||
bucket,
|
||||
"object.bin",
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create data movement upload");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_unreadable_ownership() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-corrupt";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let mut corrupted = 0;
|
||||
for temp_dir in &temp_dirs {
|
||||
for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
tokio::fs::write(path, b"invalid upload metadata")
|
||||
.await
|
||||
.expect("corrupt upload metadata");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "upload metadata must exist before corruption");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("unreadable upload ownership must block cleanup");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
|
||||
@@ -46,9 +46,9 @@ use super::super::{
|
||||
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
|
||||
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
|
||||
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, delete_file_info_version_id,
|
||||
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
|
||||
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer_with_orig_volume, debug,
|
||||
delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
|
||||
finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
|
||||
@@ -4184,7 +4184,19 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4203,7 +4215,7 @@ impl SetDisks {
|
||||
// The owned publication capability supplies the exact target write
|
||||
// domain at commit time; the generic PUT path must not reacquire it.
|
||||
opts.no_lock = true;
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4214,8 +4226,9 @@ impl SetDisks {
|
||||
data: &'a mut PutObjReader,
|
||||
opts: &'a ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
|
||||
}
|
||||
|
||||
async fn put_object_with_old_current_size_inner(
|
||||
@@ -4225,6 +4238,7 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
mut publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
|
||||
let protect_write = opts.shard_integrity_write_enabled();
|
||||
@@ -4459,9 +4473,10 @@ impl SetDisks {
|
||||
if let Some(disk) = disk_op
|
||||
&& disk.is_online().await
|
||||
{
|
||||
match create_bitrot_writer(
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
is_inline_buffer,
|
||||
Some(disk),
|
||||
bucket,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_obj,
|
||||
shard_file_size,
|
||||
@@ -4533,7 +4548,7 @@ impl SetDisks {
|
||||
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
|
||||
);
|
||||
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
|
||||
@@ -4788,6 +4803,86 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
// A new version has no outer bucket snapshot. Prepare its remote
|
||||
// owners after staging, before taking lifecycle or object locks.
|
||||
let prepare_new_version = reservations.is_none()
|
||||
&& !opts.overwrites_existing_version()
|
||||
&& !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& write_context.acquires_namespace()
|
||||
&& write_context.allows_early_ack(opts.write_completion)
|
||||
&& publication_fence.is_none()
|
||||
&& opts.scanner_publication_commit_scope.is_none()
|
||||
&& opts.decommission_capacity_admission.is_none()
|
||||
&& opts.decommission_capacity.is_none()
|
||||
&& opts.quota_admission.is_none()
|
||||
&& opts.object_lock_config_snapshot.is_none()
|
||||
&& opts.bucket_lifecycle_lock_fence.is_none()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& opts
|
||||
.put_object_cancellation
|
||||
.as_ref()
|
||||
.is_none_or(|token| !token.is_cancelled())
|
||||
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
|
||||
.await
|
||||
.is_some();
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if prepare_new_version {
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
}
|
||||
|
||||
// Destructive writes prepare before the bucket snapshot. Local-only
|
||||
// fixtures without a metadata store can prepare before the object lock.
|
||||
let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& write_context.acquires_namespace()
|
||||
&& write_context.allows_early_ack(opts.write_completion)
|
||||
&& publication_fence.is_none()
|
||||
&& opts.scanner_publication_commit_scope.is_none()
|
||||
&& (opts.decommission_capacity_admission.is_none() || reservations.is_some())
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
if let Some(prepared) = reservations {
|
||||
crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
|
||||
} else if prepare_new_version && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil()) {
|
||||
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
|
||||
&shuffle_disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
&self.ctx,
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
} else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
|
||||
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
|
||||
.await
|
||||
.is_none()
|
||||
&& let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
|
||||
{
|
||||
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
|
||||
&shuffle_disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
&self.ctx,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Conditional writes need the exclusive fixed domain for an
|
||||
// atomic cross-pool precondition check at publication time.
|
||||
let ordered_put = prepared_metadata_reservations.is_some() && opts.http_preconditions.is_none();
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -4800,6 +4895,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
!write_context.acquires_namespace() || object_lock_guard.is_some(),
|
||||
ordered_put,
|
||||
DecommissionCapacityAdmission::Mutation,
|
||||
)
|
||||
.await?;
|
||||
@@ -4811,7 +4907,9 @@ impl SetDisks {
|
||||
|| (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered)
|
||||
{
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
if !prepare_new_version {
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
}
|
||||
if let Some(expected_incarnation_id) = source_bucket_incarnation_id
|
||||
&& opts.bucket_lifecycle_lock_fence.is_none()
|
||||
{
|
||||
@@ -4846,10 +4944,21 @@ impl SetDisks {
|
||||
let namespace_lock_acquired_to_commit_ready_started = rustfs_io_metrics::put_stage_timer();
|
||||
let commit_ready_precondition_and_timestamp_started = rustfs_io_metrics::put_stage_timer();
|
||||
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||
{
|
||||
return Err(err);
|
||||
if opts.http_preconditions.is_some() {
|
||||
if decommission_object_lock_guard.is_some()
|
||||
&& !opts.data_movement
|
||||
&& !opts.replication_request
|
||||
&& !opts.no_lock
|
||||
&& !opts.skip_decommissioned
|
||||
&& !opts.skip_rebalancing
|
||||
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
|
||||
&& let Some(err) = store.check_write_precondition(bucket, object, opts, true).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
if bucket == RUSTFS_META_BUCKET
|
||||
@@ -5267,6 +5376,11 @@ impl SetDisks {
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
let commit_object_lock_snapshot = opts
|
||||
.object_lock_config_snapshot
|
||||
.as_ref()
|
||||
.filter(|snapshot| snapshot.owns_lifecycle_guard())
|
||||
.cloned();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
@@ -5280,6 +5394,26 @@ impl SetDisks {
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
|
||||
|| commit_object_lock_snapshot
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
|
||||
let reservation_ctx = (commit_allows_early_ack
|
||||
&& write_context.acquires_namespace()
|
||||
&& commit_object_lock_guard.is_some()
|
||||
&& (commit_decommission_object_lock_guard.is_none() || ordered_put)
|
||||
&& (commit_decommission_capacity_guard.is_none() || prepared_metadata_reservations.is_some())
|
||||
&& commit_publication_guard.is_none()
|
||||
&& !quota_mutation_fence
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& lifecycle_owned)
|
||||
.then(|| Arc::clone(&self.ctx));
|
||||
let prepared_metadata_reservations = if reservation_ctx.is_some() {
|
||||
prepared_metadata_reservations
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Full-tail callers also transfer owned guards to the coordinator:
|
||||
// cancelling their ACK waiter must not cancel an in-flight rename.
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
@@ -5362,7 +5496,7 @@ impl SetDisks {
|
||||
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_capacity_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
@@ -5436,7 +5570,7 @@ impl SetDisks {
|
||||
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_capacity_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
@@ -5603,6 +5737,7 @@ impl SetDisks {
|
||||
)
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone())
|
||||
.with_reservation_context(reservation_ctx, prepared_metadata_reservations)
|
||||
.with_namespace_commit_guard(
|
||||
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
),
|
||||
@@ -5626,6 +5761,7 @@ impl SetDisks {
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
|
||||
}
|
||||
let mut rename_guard_release = None;
|
||||
let mut committed_tail_guard = None;
|
||||
let mut needs_immediate_heal = false;
|
||||
let mut tail_owns_tmp_cleanup = false;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -5657,11 +5793,27 @@ impl SetDisks {
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
|
||||
None
|
||||
} else {
|
||||
_object_lock_guard
|
||||
.take()
|
||||
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
|
||||
};
|
||||
committed_tail_guard = object_lock_guard.clone();
|
||||
let publication_guard = _publication_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
let decommission_capacity_guard = _decommission_capacity_guard.take();
|
||||
// The quorum committed this snapshot. Receiver-owned
|
||||
// fences keep its bucket generation protected while
|
||||
// the remaining ordered disk writes drain.
|
||||
let object_lock_snapshot = if rename_commit.tail_has_owned_bucket_fences {
|
||||
drop(commit_object_lock_snapshot);
|
||||
None
|
||||
} else {
|
||||
commit_object_lock_snapshot
|
||||
};
|
||||
let cleanup_bucket = commit_bucket.clone();
|
||||
let cleanup_object = commit_object.clone();
|
||||
let heal_set = commit_set.clone();
|
||||
@@ -5682,6 +5834,7 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5703,6 +5856,7 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5716,6 +5870,11 @@ impl SetDisks {
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
drop(object_lock_snapshot);
|
||||
// Every rename has settled; the remaining cleanup
|
||||
// keeps the capacity fence but no longer publishes
|
||||
// object metadata.
|
||||
drop(decommission_object_lock_guard);
|
||||
cleanup_runtime::run(cleanup_reservation, async move {
|
||||
let zero_target_cleanup = targets.is_empty();
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -5762,7 +5921,6 @@ impl SetDisks {
|
||||
tmp_delete_started,
|
||||
);
|
||||
let guard_drop_started = rustfs_io_metrics::put_stage_timer();
|
||||
drop(decommission_object_lock_guard);
|
||||
drop(decommission_capacity_guard);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DECOMMISSION_GUARD_DROP,
|
||||
@@ -5971,6 +6129,13 @@ impl SetDisks {
|
||||
lock_held_post_rename_started,
|
||||
);
|
||||
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Some(guard) = committed_tail_guard.take() {
|
||||
// The committed tail still owns every physical rename. Convert
|
||||
// only after metadata invalidation and before acknowledging PUT.
|
||||
// Unsupported peers keep their exclusive leases and the tail
|
||||
// retains ownership until its physical work has drained.
|
||||
let _ = guard.lock().await.downgrade_committed_tail().await;
|
||||
}
|
||||
if let Some(release) = rename_guard_release.take() {
|
||||
let _ = release.send(true);
|
||||
}
|
||||
@@ -6128,7 +6293,11 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if !tail_owns_tmp_cleanup {
|
||||
let all_local_disks_cleaned_tmp = all_commit_disks_local
|
||||
&& online_disks.len() == commit_set.set_drive_count
|
||||
&& online_disks.iter().all(Option::is_some)
|
||||
&& !issue3031_diag_enabled();
|
||||
if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
|
||||
let cleanup_set = commit_set.clone();
|
||||
let cleanup_tmp_dir = commit_tmp_dir.clone();
|
||||
cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
|
||||
@@ -8066,6 +8235,8 @@ fn transaction_fencing_gate_requested_for(requested: bool, fleet_confirmed: bool
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PutObjectCommitPause {
|
||||
#[cfg(test)]
|
||||
BeforeBucketSnapshot,
|
||||
BeforeNamespace,
|
||||
AfterNamespace,
|
||||
AfterQuotaReservation,
|
||||
@@ -8160,7 +8331,7 @@ impl Drop for PutObjectCommitBarrier {
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) {
|
||||
pub(crate) async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) {
|
||||
let barrier = {
|
||||
let mut slot = PUT_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
@@ -11064,7 +11235,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if let Some(err) = &gerr
|
||||
&& goi.name.is_empty()
|
||||
{
|
||||
if should_force_delete_marker_for_missing_version(&opts) {
|
||||
if (is_err_object_not_found(err) || is_err_version_not_found(err))
|
||||
&& should_force_delete_marker_for_missing_version(&opts)
|
||||
{
|
||||
version_found = false;
|
||||
} else {
|
||||
if is_err_object_not_found(err) {
|
||||
@@ -22174,6 +22347,117 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-lifecycle";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer_set = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
writer_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename tail pauses");
|
||||
put.await.expect("PUT joins").expect("PUT acknowledged");
|
||||
let mut bucket_delete =
|
||||
Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
|
||||
let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let guard = match early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early bucket lock result"));
|
||||
panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
|
||||
}
|
||||
Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
|
||||
.await
|
||||
.expect("lifecycle lock released after tail")
|
||||
.expect("bucket delete acquires lock"),
|
||||
};
|
||||
drop(guard);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-tail-read";
|
||||
let object = "early-ack-tail-read-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let put_set = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
put_set
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("tail disk should pause during rename");
|
||||
put.await
|
||||
.expect("PUT task should join")
|
||||
.expect("PUT should be acknowledged at quorum");
|
||||
let result = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await;
|
||||
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
|
||||
let mut body = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
|
||||
.await
|
||||
.expect("object body should not wait for the tail")
|
||||
.expect("object body should drain");
|
||||
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
rename_barrier.release();
|
||||
drop(rename_barrier);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
|
||||
@@ -22246,6 +22530,200 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_does_not_timeout_the_next_put() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-next-writer";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_set = Arc::clone(&set);
|
||||
let first_opts = opts.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first rename paused");
|
||||
first.await.expect("first PUT joins").expect("first PUT acknowledged");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let second = second.expect("second PUT finishes").expect("second PUT must not time out");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both rename tails must drain");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read newest object");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read full body");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
let expected_etag = second.etag.as_ref().expect("second PUT ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("each disk must retain the latest version");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_cannot_restore_a_deleted_version() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-version-delete";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put_opts = opts.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
writer
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename pauses");
|
||||
let version = put
|
||||
.await
|
||||
.expect("PUT joins")
|
||||
.expect("PUT succeeds")
|
||||
.version_id
|
||||
.expect("version ID")
|
||||
.to_string();
|
||||
let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
|
||||
let delete_set = Arc::clone(&set);
|
||||
let mut delete_opts = opts;
|
||||
delete_opts.version_id = Some(version.clone());
|
||||
let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
|
||||
delete_barrier.wait_until_paused().await;
|
||||
delete_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !delete_barrier.namespace_acquired() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("version delete acquires namespace while old tail is paused");
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
delete.await.expect("delete joins").expect("version delete succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
),
|
||||
"late tail must not restore the deleted version"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {
|
||||
|
||||
@@ -165,6 +165,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock,
|
||||
false,
|
||||
DecommissionCapacityAdmission::Mutation,
|
||||
)
|
||||
.await?
|
||||
@@ -275,6 +276,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock,
|
||||
false,
|
||||
DecommissionCapacityAdmission::Mutation,
|
||||
)
|
||||
.await?
|
||||
|
||||
@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
|
||||
WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
|
||||
WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -523,7 +523,22 @@ impl ECStore {
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
|
||||
}
|
||||
|
||||
/// Validate an external fence and the bucket incarnation in one metadata transaction.
|
||||
pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
@@ -2279,8 +2294,17 @@ mod tests {
|
||||
drop(
|
||||
lock.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("fixture rename tail should finish before checking its generation"),
|
||||
.expect("fixture object lock should be available"),
|
||||
);
|
||||
// A quorum-acknowledged rename can release the object lock while its
|
||||
// physical tail still holds the instance-wide namespace owner.
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while ecstore.ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("fixture namespace owners should drain before checking its generation");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_put.saturating_add(3),
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
use crate::error::{Result, StorageError};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use rustfs_lock::distributed_lock::LockLostSignal;
|
||||
@@ -65,7 +66,7 @@ struct RegisteredGuard {
|
||||
|
||||
enum LockLossProbe {
|
||||
Distributed(Arc<LockLostSignal>),
|
||||
Local,
|
||||
Local(Option<rustfs_lock::fast_lock::guard::FastLockGuardProbe>),
|
||||
#[cfg(test)]
|
||||
Test(Arc<std::sync::atomic::AtomicBool>),
|
||||
}
|
||||
@@ -74,14 +75,19 @@ impl LockLossProbe {
|
||||
fn from_guard(guard: &NamespaceLockGuard) -> Self {
|
||||
match guard.lock_lost_signal() {
|
||||
Some(signal) => Self::Distributed(signal),
|
||||
None => Self::Local,
|
||||
None => Self::Local(match guard {
|
||||
NamespaceLockGuard::Fast(guard) => guard.release_probe(),
|
||||
NamespaceLockGuard::Standard(_) => None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lost(&self) -> bool {
|
||||
// Registry callers hold entries before checking a local shard; shard
|
||||
// operations never acquire the bucket fence registry.
|
||||
match self {
|
||||
Self::Distributed(signal) => signal.is_lost(),
|
||||
Self::Local => false,
|
||||
Self::Local(probe) => probe.as_ref().is_some_and(|probe| probe.is_lost()),
|
||||
#[cfg(test)]
|
||||
Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst),
|
||||
}
|
||||
@@ -98,9 +104,74 @@ pub(super) struct FenceRegistration {
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BucketFenceRegistry {
|
||||
entries: Mutex<HashMap<String, FenceEntry>>,
|
||||
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ReservationFenceSlot {
|
||||
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
|
||||
}
|
||||
|
||||
/// Shared by short bursts of reservations, but never kept alive by the cache.
|
||||
pub(super) struct ReservationFenceGuard {
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
created_at: tokio::time::Instant,
|
||||
_slot: Arc<ReservationFenceSlot>,
|
||||
}
|
||||
|
||||
impl ReservationFenceGuard {
|
||||
pub(super) fn new(
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
slot: Arc<ReservationFenceSlot>,
|
||||
) -> Self {
|
||||
Self {
|
||||
publication,
|
||||
lifecycle,
|
||||
created_at: tokio::time::Instant::now(),
|
||||
_slot: slot,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_reusable(&self) -> bool {
|
||||
// Stop admitting new reservations through this shared reader so a
|
||||
// queued bucket writer can progress after the existing tails drain.
|
||||
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
|
||||
}
|
||||
|
||||
fn is_held(&self) -> bool {
|
||||
!self.publication.is_lock_lost()
|
||||
&& !self.publication.is_released()
|
||||
&& !self.lifecycle.is_lock_lost()
|
||||
&& self
|
||||
.lifecycle
|
||||
.namespace_lock_guard()
|
||||
.is_some_and(|guard| !guard.is_released())
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
|
||||
fn is_valid(&self) -> bool {
|
||||
self.is_held()
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketFenceRegistry {
|
||||
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
|
||||
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
|
||||
let key = (bucket.to_owned(), incarnation);
|
||||
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
|
||||
return slot;
|
||||
}
|
||||
if slots.len() >= 1024 {
|
||||
slots.retain(|_, slot| slot.strong_count() != 0);
|
||||
}
|
||||
let slot = Arc::new(ReservationFenceSlot::default());
|
||||
slots.insert(key, Arc::downgrade(&slot));
|
||||
slot
|
||||
}
|
||||
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
|
||||
@@ -234,7 +305,9 @@ impl BucketIncarnationFenceGuard {
|
||||
}
|
||||
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
self.inner
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_lock_guard(&self) -> Option<&NamespaceLockGuard> {
|
||||
@@ -352,6 +425,67 @@ mod tests {
|
||||
reg.exit("b", fourth.token, false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_local_lock_does_not_add_a_namespace_fence() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Disabled(
|
||||
rustfs_lock::fast_lock::DisabledLockManager::new(),
|
||||
));
|
||||
let lock = NamespaceLock::with_local_manager("disabled-fence".into(), manager);
|
||||
let guard = lock
|
||||
.get_read_lock(ObjectKey::new("b", "object"), "owner", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("disabled lock");
|
||||
let mut opts = crate::object_api::ObjectOptions::default();
|
||||
opts.add_namespace_lock_guard(&guard);
|
||||
assert!(opts.namespace_lock_fence.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_local_fence_loss_invalidates_live_incarnation_memo() {
|
||||
let manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
|
||||
let global = Arc::new(rustfs_lock::GlobalLockManager::Enabled(manager.clone()));
|
||||
let lock = NamespaceLock::with_local_manager("forced-fence".into(), global);
|
||||
let key = ObjectKey::new("b", "lifecycle");
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
let inner = lock
|
||||
.get_read_lock(key.clone(), "first", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("first lock");
|
||||
let mut first = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
|
||||
first.validate(true, || async { Ok(uuid(7)) }).await.expect("seed memo");
|
||||
let mut opts = crate::object_api::ObjectOptions::default();
|
||||
first.attach_to_object_options(&mut opts);
|
||||
opts.add_namespace_lock_guard(first.namespace_lock_guard().expect("local guard"));
|
||||
assert!(
|
||||
!opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("lifecycle fence")
|
||||
.is_lock_lost()
|
||||
);
|
||||
assert!(!opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
assert!(
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.expect("attached fence")
|
||||
.is_lock_lost()
|
||||
);
|
||||
assert!(opts.namespace_lock_fence.as_ref().expect("namespace fence").is_lock_lost());
|
||||
let writer = lock
|
||||
.get_write_lock(key.clone(), "writer", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("writer");
|
||||
drop(writer);
|
||||
let inner = lock
|
||||
.get_read_lock(key, "second", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("second lock");
|
||||
let mut second = BucketIncarnationFenceGuard::new(registry.clone(), "b", inner, true);
|
||||
assert_eq!(second.validate(true, || async { Ok(uuid(8)) }).await.expect("reload"), uuid(8));
|
||||
assert!(first.validate(true, || async { Ok(uuid(7)) }).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fence_guard_forwards_distributed_lock_loss_to_registry() {
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
|
||||
@@ -5340,6 +5340,203 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn conditional_multipart_must_not_commit_against_an_older_readable_pool() {
|
||||
use crate::storage_api_contracts::multipart::CompletePart;
|
||||
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create conditional multipart test directory");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "conditional-mpu-unreadable", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("conditional-mpu-{}", uuid::Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create conditional multipart test bucket");
|
||||
let mut old_body = PutObjReader::from_vec(b"older-pool".to_vec());
|
||||
let old = store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write old object to staging pool");
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload in pool 0");
|
||||
let mut part_body = PutObjReader::from_vec(b"stale-writer".to_vec());
|
||||
let part = store.pools[0]
|
||||
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart part in pool 0");
|
||||
let mut new_body = PutObjReader::from_vec(b"newer-pool".to_vec());
|
||||
let new = store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut new_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer object to pool 1");
|
||||
let old_etag = old.etag.clone().expect("old object must have an ETag");
|
||||
let before = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
|
||||
let mut metadata = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let path = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/xl.meta"));
|
||||
let original = tokio::fs::read(&path)
|
||||
.await
|
||||
.expect("save original object metadata before fault injection");
|
||||
metadata.push((path, original));
|
||||
}
|
||||
let mut fault_error = None;
|
||||
for (path, _) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, b"invalid object metadata").await {
|
||||
fault_error = Some(err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut fault_contents = Vec::new();
|
||||
if fault_error.is_none() {
|
||||
for (path, _) in &metadata {
|
||||
fault_contents.push(tokio::fs::read(path).await);
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
}
|
||||
let unreadable = if fault_error.is_none() {
|
||||
Some(
|
||||
store.pools[1]
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed = if fault_error.is_none() {
|
||||
let completion_store = Arc::clone(&store);
|
||||
let completion_bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let part_etag = part.etag.clone();
|
||||
Some(
|
||||
tokio::spawn(async move {
|
||||
completion_store
|
||||
.complete_multipart_upload(
|
||||
&completion_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part_etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(old_etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut restore_error = None;
|
||||
for (path, original) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, original).await {
|
||||
restore_error = Some(err);
|
||||
}
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
let after = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
let recovered_body = match store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(mut reader) => {
|
||||
let mut body = Vec::new();
|
||||
let result = reader
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.map(|_| body)
|
||||
.map_err(|err| err.to_string());
|
||||
drop(reader);
|
||||
result
|
||||
}
|
||||
Err(err) => Err(err.to_string()),
|
||||
};
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
|
||||
assert!(fault_error.is_none(), "failed to inject metadata fault: {fault_error:?}");
|
||||
assert!(restore_error.is_none(), "failed to restore metadata: {restore_error:?}");
|
||||
assert_eq!(fault_contents.len(), 4, "all four pool 1 metadata files must be inspected");
|
||||
for observed in fault_contents {
|
||||
assert_eq!(
|
||||
observed.expect("read injected metadata"),
|
||||
b"invalid object metadata",
|
||||
"each pool 1 metadata file must contain the injected corruption"
|
||||
);
|
||||
}
|
||||
assert_eq!(before.expect("latest object before fault").etag, new.etag);
|
||||
assert_ne!(old.etag, new.etag, "fixture bodies must produce different ETags");
|
||||
let read_error = unreadable
|
||||
.expect("metadata fault should have been injected")
|
||||
.expect_err("the newer pool must fail its object metadata read during the fault");
|
||||
assert!(
|
||||
!crate::error::is_err_object_not_found(&read_error) && !crate::error::is_err_version_not_found(&read_error),
|
||||
"corrupt metadata must produce a non-not-found read error: {read_error:?}"
|
||||
);
|
||||
let completed = completed
|
||||
.expect("completion should have been attempted")
|
||||
.expect("completion task should not panic");
|
||||
let Err(completion_error) = completed else {
|
||||
panic!("stale If-Match must not commit while the newer object's metadata is unreadable");
|
||||
};
|
||||
assert_eq!(completion_error, read_error, "completion must propagate the unreadable pool's error");
|
||||
assert_eq!(after.expect("newer object after recovery").etag, new.etag);
|
||||
assert_eq!(recovered_body.expect("newer body after recovery"), b"newer-pool");
|
||||
let parts = parts.expect("failed completion must leave the upload retryable");
|
||||
assert_eq!(parts.parts.len(), 1);
|
||||
assert_eq!(parts.parts[0].etag, part.etag);
|
||||
}
|
||||
|
||||
async fn migrate_versioned_decommission_test_object(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -37,8 +37,8 @@ use crate::error::{
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::list::{
|
||||
ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
|
||||
StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -80,6 +80,21 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthoritativeListOperations for ECStore {
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
self.walk_authoritatively(rx, bucket, prefix, result, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the versioned listing options used by recursive object walks.
|
||||
///
|
||||
/// `list_path_result` normally derives these scan bounds before dispatching to
|
||||
@@ -1292,6 +1307,37 @@ impl ECStore {
|
||||
);
|
||||
|
||||
let rx = rx.child_token();
|
||||
if self.pools.len() == 1 && self.pools[0].disk_set.len() == 1 {
|
||||
rustfs_io_metrics::record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 1, 1);
|
||||
let res = self.pools[0].disk_set[0].list_path(rx, opts, sender).await;
|
||||
let mut outcome = ListPathRawOutcome::default();
|
||||
let err = match res {
|
||||
Ok(set_outcome) => {
|
||||
outcome.limit_reached = set_outcome.limit_reached;
|
||||
None
|
||||
}
|
||||
Err(err) => Some(err),
|
||||
};
|
||||
let errs = [err];
|
||||
if is_all_not_found(&errs) {
|
||||
return Ok(outcome);
|
||||
}
|
||||
if let Some(err) = errs.into_iter().flatten().next() {
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"store_list_objects_list_merged",
|
||||
merge_started.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
debug!(
|
||||
set_pair_count = 1,
|
||||
limit_reached = outcome.limit_reached,
|
||||
error_count = 0,
|
||||
"store list_merged finished"
|
||||
);
|
||||
return Ok(outcome);
|
||||
}
|
||||
|
||||
let mut futures = Vec::new();
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
@@ -1519,6 +1565,57 @@ impl ECStore {
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn walk_authoritatively(
|
||||
self: Arc<Self>,
|
||||
cancellation: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
let cancel = cancellation.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let mut scans = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
inputs.push(receiver);
|
||||
let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
|
||||
// Internal absence checks must include reserved object prefixes.
|
||||
scan.skip_hidden_prefix_check = true;
|
||||
scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
|
||||
}
|
||||
}
|
||||
if scans.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
|
||||
let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
|
||||
let scan_results = join_all(scans).await;
|
||||
let merge_result = merge.await.map_err(Error::from)?;
|
||||
let forward_result = forwarder.await.map_err(Error::from);
|
||||
|
||||
// An unavailable set cannot establish absence, even when another set
|
||||
// completed. Only an explicitly missing directory or volume is empty.
|
||||
for scan in scan_results {
|
||||
match scan {
|
||||
Ok(outcome) if !outcome.limit_reached => {}
|
||||
Ok(_) => return Err(StorageError::ErasureReadQuorum),
|
||||
Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
merge_result?;
|
||||
forward_result?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(Error::other("authoritative object walk was cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
@@ -1811,7 +1908,7 @@ async fn gather_results(
|
||||
if opts.versioned
|
||||
&& is_object
|
||||
&& entry
|
||||
.xl_meta()
|
||||
.xl_meta_ref()
|
||||
.is_ok_and(|meta| !meta.versions.is_empty() && meta.versions.iter().all(|version| version.header.free_version()))
|
||||
{
|
||||
continue;
|
||||
@@ -2489,7 +2586,8 @@ impl SetDisks {
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
/// version is resolved at the read quorum of its own erasure layout; a
|
||||
/// version uses the read quorum of its own erasure layout. Null versions
|
||||
/// in the system bucket also retain GET's commit-proof selection. A
|
||||
/// version that cannot be ruled out fails the scan instead of being
|
||||
/// omitted. Merged `ListObjectVersions` pages go through `list_path`.
|
||||
async fn list_versions_authoritatively(
|
||||
@@ -2828,6 +2926,7 @@ mod test {
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
|
||||
use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use bytes::Bytes;
|
||||
@@ -5688,6 +5787,126 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
for new_etag in ["new", "old"] {
|
||||
let new =
|
||||
test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
|
||||
let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
|
||||
|
||||
assert!(matches!(
|
||||
SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_selects_committed_null_version() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
|
||||
.collect(),
|
||||
);
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
|
||||
.expect("committed generation should resolve")
|
||||
.expect("committed record should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (old_copies, new_copies, expected) in [
|
||||
(2, 1, Some("old")),
|
||||
(1, 2, None),
|
||||
(0, 3, Some("new")),
|
||||
(0, 2, Some("new")),
|
||||
(0, 1, None),
|
||||
] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| {
|
||||
if slot < old_copies {
|
||||
Some(old.clone())
|
||||
} else if slot < old_copies + new_copies {
|
||||
Some(new.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let errors = (0..4)
|
||||
.map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
|
||||
.collect::<Vec<_>>();
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
|
||||
|
||||
if let Some(expected) = expected {
|
||||
let info = resolved
|
||||
.expect("degraded committed record should resolve")
|
||||
.expect("degraded committed record should be present")
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
} else {
|
||||
assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for old_copies in [1, 2] {
|
||||
let entries = MetaCacheEntries(vec![
|
||||
Some(new.clone()),
|
||||
Some(new.clone()),
|
||||
Some(old.clone()),
|
||||
(old_copies == 2).then(|| old.clone()),
|
||||
]);
|
||||
let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
|
||||
.expect("user bucket read quorum should resolve")
|
||||
.expect("user bucket record should be present");
|
||||
let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
|
||||
let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
|
||||
let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
|
||||
.expect("explicit version read quorum should resolve")
|
||||
.expect("explicit version should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
|
||||
let entry = test_object_meta_entry_with_erasure_versions(
|
||||
@@ -6029,6 +6248,49 @@ mod test {
|
||||
Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
|
||||
let (_dirs, store) = single_set_store(16).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for (slot, disk) in set.disks.read().await.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("fixture drive is online");
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{name}/{STORAGE_FORMAT_FILE}"),
|
||||
versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
|
||||
)
|
||||
.await
|
||||
.expect("persist reserved metadata versions");
|
||||
}
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let walk = store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let collect = async {
|
||||
let mut versions = Vec::new();
|
||||
while let Some(item) = receiver.recv().await {
|
||||
assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
|
||||
let item = item.item.expect("walk must return an object");
|
||||
versions.push((item.name, item.version_id));
|
||||
}
|
||||
versions
|
||||
};
|
||||
let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
|
||||
.await
|
||||
.expect("authoritative reserved-prefix walk must terminate");
|
||||
walk.expect("authoritative walk must succeed");
|
||||
assert_eq!(
|
||||
versions,
|
||||
[
|
||||
(name.to_string(), Some(Uuid::from_u128(1))),
|
||||
(name.to_string(), Some(Uuid::from_u128(2)))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// A versioned DeleteObject(versionId) can reach write quorum while three
|
||||
// drives are offline; they keep the deleted version after they return. A
|
||||
// different drive going offline later must not turn that sub-quorum
|
||||
@@ -6099,6 +6361,16 @@ mod test {
|
||||
.await
|
||||
.expect("set walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
let walk =
|
||||
store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let drain = async { while receiver.recv().await.is_some() {} };
|
||||
let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
|
||||
.await
|
||||
.expect("authoritative store walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let recursive_delete_scan = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)
|
||||
|
||||
@@ -456,6 +456,7 @@ mod rebalance;
|
||||
pub(crate) use rebalance::PoolSpaceForTest;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub(crate) use peer::admit_local_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
@@ -498,6 +499,17 @@ pub struct ECStore {
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
}
|
||||
|
||||
impl Drop for ECStore {
|
||||
/// Clear the instance context's `BucketMetadataSys` when its backing store
|
||||
/// is dropped. Without this, the shared bootstrap context retains a
|
||||
/// `BucketMetadataSys` whose `Weak<ECStore>` is dangling after the store
|
||||
/// goes away (e.g. between sequential test fixtures that share the
|
||||
/// bootstrap), and the next `object_store()` call panics.
|
||||
fn drop(&mut self) {
|
||||
self.ctx.clear_bucket_metadata_sys_if_stale();
|
||||
}
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
|
||||
|
||||
@@ -19,10 +19,28 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_utils::path::encode_dir_object_ref;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
|
||||
type Error = Error;
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
let (_, lifecycle_guard) = self
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await?;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.ensure_no_multipart_uploads(bucket).await?;
|
||||
}
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
|
||||
std::sync::OnceLock::new();
|
||||
@@ -612,7 +630,7 @@ impl ECStore {
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts).await
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts, false).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1154,12 +1172,31 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option<StorageError> {
|
||||
pub(crate) async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Option<StorageError> {
|
||||
let mut lookup_opts = opts.clone();
|
||||
let http_preconditions = lookup_opts.http_preconditions?;
|
||||
lookup_opts.http_preconditions = None;
|
||||
lookup_opts.no_lock = true;
|
||||
let oi = self.get_object_info_for_delete(bucket, object, &lookup_opts).await;
|
||||
let oi = if require_all_pools {
|
||||
lookup_opts.metadata_cache_safe = false;
|
||||
lookup_opts.suppress_read_repair = true;
|
||||
// Only completion uses this branch, after validating bucket and object.
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
|
||||
.await
|
||||
.and_then(|(info, _)| {
|
||||
lookup_opts.precondition_check(&info)?;
|
||||
Ok(info)
|
||||
})
|
||||
} else {
|
||||
self.get_object_info_for_delete(bucket, object, &lookup_opts).await
|
||||
};
|
||||
crate::set_disk::SetDisks::evaluate_write_precondition(oi, &http_preconditions, bucket, object, opts)
|
||||
}
|
||||
|
||||
@@ -1186,11 +1223,13 @@ impl ECStore {
|
||||
.await;
|
||||
return enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await;
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||
{
|
||||
return Err(err);
|
||||
if opts.http_preconditions.is_some() && !opts.data_movement {
|
||||
// Preserve explicitly scoped internal reads and borrowed-lock callers.
|
||||
let require_all_pools =
|
||||
!opts.replication_request && !opts.no_lock && !opts.skip_decommissioned && !opts.skip_rebalancing;
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts, require_all_pools).await {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
let result = match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -85,6 +85,230 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Prepare an inert user-object reservation on the currently admitted disk.
|
||||
/// An unclaimed token expires and does not hold the object's metadata lock.
|
||||
#[cfg(test)]
|
||||
pub async fn reserve_local_object_metadata(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare before the caller acquires bucket lifecycle or object locks.
|
||||
/// The token holds the bucket publication fence through physical completion.
|
||||
pub async fn reserve_local_object_metadata_at_incarnation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_at_incarnation_with_ttl(disk_ref, bucket, object, incarnation, timeout, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare a short-lived token for peers that may abandon the response.
|
||||
pub async fn reserve_local_object_metadata_short(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_at_incarnation_with_ttl(
|
||||
disk_ref,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
timeout,
|
||||
Some(std::time::Duration::from_secs(2)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_at_incarnation_with_ttl(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
ttl: Option<std::time::Duration>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if self.ctx.lock_manager().is_disabled()
|
||||
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|
||||
|| is_meta_bucketname(bucket)
|
||||
|| incarnation.is_nil()
|
||||
|| timeout.is_zero()
|
||||
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
|
||||
let mut cached = slot.owner.lock().await;
|
||||
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
|
||||
Some(owner) => owner,
|
||||
None => {
|
||||
// Bucket deletion takes publication before lifecycle and metadata.
|
||||
let lock = self
|
||||
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let publication = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
// Older peers may acquire lifecycle without publication.
|
||||
let lifecycle = self
|
||||
.acquire_bucket_incarnation_fence(bucket, incarnation)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
|
||||
if !owner.is_reusable() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
*cached = Arc::downgrade(&owner);
|
||||
owner
|
||||
}
|
||||
};
|
||||
drop(cached);
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner), ttl)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_with_owner(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
|
||||
ttl: Option<std::time::Duration>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
match ttl {
|
||||
Some(ttl) => {
|
||||
wrapper
|
||||
.prepare_object_metadata_with_owner_ttl(bucket, object, owner, ttl)
|
||||
.await
|
||||
}
|
||||
None => wrapper.prepare_object_metadata_with_owner(bucket, object, owner).await,
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
/// Refresh an inert token on its original disk and object.
|
||||
pub async fn renew_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.renew_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token on its original disk and object.
|
||||
pub async fn activate_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed token on the currently admitted disk.
|
||||
/// A token already claimed by a physical mutation cannot be cancelled.
|
||||
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
Ok(wrapper.cancel_object_metadata_reservation(token))
|
||||
}
|
||||
|
||||
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
|
||||
/// The token is bound to the admitted disk instance and destination. A claimed
|
||||
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
|
||||
/// The caller must retain the bucket lifecycle fence until the physical tail
|
||||
/// finishes; the reservation does not identify a bucket incarnation.
|
||||
pub async fn rename_local_data_reserved(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
token: Uuid,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if token.is_nil()
|
||||
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|
||||
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|
||||
|| is_meta_bucketname(destination.0)
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
@@ -270,6 +494,7 @@ impl ECStore {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -351,7 +576,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
pub(crate) async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
@@ -783,6 +1008,116 @@ mod tests {
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let dir = tempfile::tempdir().expect("disk root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, dir.path(), id).await;
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("object", version, b"reserved");
|
||||
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
|
||||
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
|
||||
.await
|
||||
.expect("user bucket");
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(
|
||||
&disk_ref,
|
||||
"target-bucket/../.rustfs.sys",
|
||||
"object",
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
let token = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve current disk");
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
store
|
||||
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
|
||||
.await
|
||||
.expect("activate token");
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
token,
|
||||
)
|
||||
.await
|
||||
.expect("reserved publication");
|
||||
assert!(
|
||||
!store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("consumed token")
|
||||
);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read reserved publication");
|
||||
assert_eq!(read.data, fi.data);
|
||||
|
||||
let stale = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve before disk replacement");
|
||||
let replacement = target_disk(&ctx, dir.path(), id).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
stale,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
));
|
||||
let after = replacement
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement retains publication");
|
||||
assert_eq!(after.data, fi.data);
|
||||
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
|
||||
@@ -604,6 +604,24 @@ impl ECStore {
|
||||
Ok(idx)
|
||||
}
|
||||
|
||||
/// A speculative hint only: callers must repeat the locked selection before committing.
|
||||
pub(super) async fn get_pool_idx_for_reservation_hint(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: true,
|
||||
skip_decommissioned: true,
|
||||
skip_rebalancing: true,
|
||||
..Default::default()
|
||||
};
|
||||
match self.get_pool_idx_existing_with_opts(bucket, object, &opts).await {
|
||||
Ok(idx) => Ok(idx),
|
||||
Err(err) if is_err_object_not_found(&err) => {
|
||||
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
|
||||
let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await {
|
||||
Ok(res) => res,
|
||||
@@ -837,7 +855,29 @@ impl ECStore {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Observe every pool for a conditional-write preflight. This does not
|
||||
/// hold the namespace locks needed to prove a condition at publication.
|
||||
pub(super) async fn get_latest_object_info_for_conditional_write(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, true).await
|
||||
}
|
||||
|
||||
async fn get_latest_object_info_with_pool_policy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned && !require_all_pools {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
Some(
|
||||
(0..self.pools.len())
|
||||
@@ -853,7 +893,7 @@ impl ECStore {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
|
||||
if opts.skip_rebalancing && !require_all_pools && self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -873,6 +913,9 @@ impl ECStore {
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
if require_all_pools && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
|
||||
return Err(e);
|
||||
}
|
||||
candidates.push(LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx,
|
||||
|
||||
@@ -2070,6 +2070,53 @@ mod test {
|
||||
assert!(meta.versions.iter().all(|version| !version.header.free_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bytes_count_toward_listing_limit_matches_fileinfo_visibility() {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut obj = FileInfo::new("object", 2, 2);
|
||||
obj.version_id = Some(Uuid::new_v4());
|
||||
obj.mod_time = Some(OffsetDateTime::now_utc());
|
||||
meta.add_version(obj.clone()).expect("add object version");
|
||||
let obj_bytes = meta.marshal_msg().expect("marshal object");
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(&obj_bytes));
|
||||
|
||||
let dm = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: obj.mod_time.map(|t| t + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
meta.delete_version(&dm).expect("add delete marker");
|
||||
let dm_bytes = meta.marshal_msg().expect("marshal delete marker");
|
||||
assert!(!FileMeta::bytes_count_toward_listing_limit(&dm_bytes));
|
||||
|
||||
let (_, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
source.version_id = Some(Uuid::new_v4());
|
||||
let mut transitioned = FileMeta::new();
|
||||
transitioned.add_version(source.clone()).expect("seed transitioned version");
|
||||
let mut del_req = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: source.version_id,
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
del_req.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
transitioned
|
||||
.delete_version(&del_req)
|
||||
.expect("delete transitioned version leaving free version");
|
||||
assert!(!FileMeta::bytes_count_toward_listing_limit(
|
||||
&transitioned.marshal_msg().expect("marshal free-only")
|
||||
));
|
||||
|
||||
transitioned.add_version(obj).expect("add live object version");
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(
|
||||
&transitioned.marshal_msg().expect("marshal free + live object")
|
||||
));
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(b"corrupt-xl-meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_uses_canonical_equal_time_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
|
||||
@@ -324,6 +324,63 @@ impl FileMeta {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn buffers_match_strict(self_buf: &[u8], other_buf: &[u8]) -> bool {
|
||||
let Ok((s_meta, _)) = Self::parse_validated_xl_envelope(self_buf) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_meta, _)) = Self::parse_validated_xl_envelope(other_buf) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if s_meta.is_empty() || o_meta.is_empty() {
|
||||
return s_meta.is_empty() && o_meta.is_empty();
|
||||
}
|
||||
|
||||
let Ok((s_len, s_header_ver, _s_meta_ver, s_meta)) = Self::decode_validated_xl_headers(s_meta) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_len, o_header_ver, _o_meta_ver, o_meta)) = Self::decode_validated_xl_headers(o_meta) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if s_len != o_len {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut s_cur: Cursor<&[u8]> = Cursor::new(s_meta);
|
||||
let mut o_cur: Cursor<&[u8]> = Cursor::new(o_meta);
|
||||
for _ in 0..s_len {
|
||||
let Ok((s_header_raw, _)) = Self::read_next_raw_slices(s_meta, &mut s_cur) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_header_raw, _)) = Self::read_next_raw_slices(o_meta, &mut o_cur) else {
|
||||
return false;
|
||||
};
|
||||
if s_header_ver == XL_HEADER_VERSION
|
||||
&& o_header_ver == XL_HEADER_VERSION
|
||||
&& s_header_raw.len() == 38
|
||||
&& o_header_raw.len() == 38
|
||||
&& s_header_raw == o_header_raw
|
||||
&& parse_canonical_v3_header_fields(s_header_raw).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut s_header = FileMetaVersionHeader::default();
|
||||
if s_header.unmarshal_v(s_header_ver, s_header_raw).is_err() {
|
||||
return false;
|
||||
}
|
||||
let mut o_header = FileMetaVersionHeader::default();
|
||||
if o_header.unmarshal_v(o_header_ver, o_header_raw).is_err() {
|
||||
return false;
|
||||
}
|
||||
if !Self::headers_match_strict(&s_header, &o_header) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn header_matches_canonical_v3_raw_strict(s_header: &FileMetaVersionHeader, buf: &[u8]) -> Option<bool> {
|
||||
if buf.len() != 38 {
|
||||
@@ -625,6 +682,49 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bytes_count_toward_listing_limit(buf: &[u8]) -> bool {
|
||||
let Ok((meta, _inline_buf)) = Self::parse_validated_xl_envelope(buf) else {
|
||||
return true;
|
||||
};
|
||||
if meta.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Ok((versions_len, header_ver, _meta_ver, meta)) = Self::decode_validated_xl_headers(meta) else {
|
||||
return true;
|
||||
};
|
||||
if versions_len == 0 {
|
||||
return false;
|
||||
}
|
||||
let mut cur = Cursor::new(meta);
|
||||
let mut selected: Option<(FileMetaVersionHeader, &[u8])> = None;
|
||||
let mut any_non_hidden = false;
|
||||
for _ in 0..versions_len {
|
||||
let Ok((header, ver_meta)) = Self::read_next_shallow_slice(meta, &mut cur, header_ver) else {
|
||||
return true;
|
||||
};
|
||||
if header.version_type != VersionType::Delete && !header.free_version() {
|
||||
any_non_hidden = true;
|
||||
}
|
||||
if selected.is_some() {
|
||||
continue;
|
||||
}
|
||||
if header.free_version() && header.version_id.is_some_and(|id| !id.is_nil()) {
|
||||
continue;
|
||||
}
|
||||
selected = Some((header, ver_meta));
|
||||
}
|
||||
let Some((header, ver_meta)) = selected else {
|
||||
return false;
|
||||
};
|
||||
if !header.free_version() && header.version_type != VersionType::Delete {
|
||||
return true;
|
||||
}
|
||||
if !any_non_hidden {
|
||||
return false;
|
||||
}
|
||||
FileMetaVersion::decode_fileinfo_from_slice(ver_meta, "", "", true, false).is_err()
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "FileMeta")]
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
let mut wr =
|
||||
|
||||
@@ -352,6 +352,22 @@ impl MetaCacheEntry {
|
||||
(prefer, true)
|
||||
}
|
||||
|
||||
pub fn matches_strict_borrowed(&mut self, other: &MetaCacheEntry) -> bool {
|
||||
if self.name != other.name {
|
||||
return false;
|
||||
}
|
||||
|
||||
if other.is_dir() || self.is_dir() {
|
||||
return other.is_dir() == self.is_dir();
|
||||
}
|
||||
|
||||
if self.cached.is_none() && other.cached.is_none() {
|
||||
return FileMeta::buffers_match_strict(&self.metadata, &other.metadata);
|
||||
}
|
||||
|
||||
self.matches_strict(other)
|
||||
}
|
||||
|
||||
pub fn matches_strict(&mut self, other: &MetaCacheEntry) -> bool {
|
||||
if self.name != other.name {
|
||||
return false;
|
||||
@@ -989,6 +1005,17 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn write_obj_buffered(&mut self, obj: &MetaCacheEntry) -> Result<()> {
|
||||
self.init().await?;
|
||||
|
||||
Self::encode_obj(&mut self.buf, obj)?;
|
||||
if self.buf.len() >= METACACHE_READ_CHUNK_SIZE {
|
||||
self.flush().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn close(&mut self) -> Result<()> {
|
||||
self.close_with_limit_reached(false).await
|
||||
}
|
||||
@@ -1282,6 +1309,85 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
|
||||
}
|
||||
|
||||
pub fn skip_current(&mut self) -> bool {
|
||||
if self.has_current && self.init && self.err.is_none() {
|
||||
self.has_current = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_peek_buffered(&mut self) -> Option<Result<Option<MetaCacheEntry>>> {
|
||||
if !self.init || self.err.is_some() || self.terminated {
|
||||
return None;
|
||||
}
|
||||
|
||||
let avail = &self.buf[self.pos..];
|
||||
let &first = avail.first()?;
|
||||
if first != 0xc3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let str_marker = *avail.get(1)?;
|
||||
let (str_hdr, str_len) = match str_marker {
|
||||
0xa0..=0xbf => (1usize, usize::from(str_marker & 0x1f)),
|
||||
0xd9 => (2usize, usize::from(*avail.get(2)?)),
|
||||
0xda => (3usize, usize::from(u16::from_be_bytes([*avail.get(2)?, *avail.get(3)?]))),
|
||||
0xdb => (
|
||||
5usize,
|
||||
usize::try_from(u32::from_be_bytes([*avail.get(2)?, *avail.get(3)?, *avail.get(4)?, *avail.get(5)?])).ok()?,
|
||||
),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let bin_offset = 1usize.checked_add(str_hdr)?.checked_add(str_len)?;
|
||||
let bin_marker = *avail.get(bin_offset)?;
|
||||
let (bin_hdr, bin_len) = match bin_marker {
|
||||
0xc4 => (2usize, usize::from(*avail.get(bin_offset + 1)?)),
|
||||
0xc5 => (
|
||||
3usize,
|
||||
usize::from(u16::from_be_bytes([*avail.get(bin_offset + 1)?, *avail.get(bin_offset + 2)?])),
|
||||
),
|
||||
0xc6 => (
|
||||
5usize,
|
||||
usize::try_from(u32::from_be_bytes([
|
||||
*avail.get(bin_offset + 1)?,
|
||||
*avail.get(bin_offset + 2)?,
|
||||
*avail.get(bin_offset + 3)?,
|
||||
*avail.get(bin_offset + 4)?,
|
||||
]))
|
||||
.ok()?,
|
||||
),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let total_len = bin_offset.checked_add(bin_hdr)?.checked_add(bin_len)?;
|
||||
if avail.len() < total_len {
|
||||
return None;
|
||||
}
|
||||
|
||||
let Ok(decoded) = from_utf8(&avail[1 + str_hdr..bin_offset]) else {
|
||||
return None;
|
||||
};
|
||||
let name = decoded.to_owned();
|
||||
let metadata = if bin_len == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
avail[bin_offset + bin_hdr..total_len].to_vec()
|
||||
};
|
||||
|
||||
self.pos += total_len;
|
||||
self.reset();
|
||||
self.has_current = true;
|
||||
Some(Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
metadata,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn skip(&mut self, size: usize) -> Result<()> {
|
||||
self.check_init().await?;
|
||||
|
||||
@@ -4257,4 +4363,95 @@ mod tests {
|
||||
.expect("resolve mixed empty");
|
||||
assert_eq!(res_mixed_empty.cached.expect("cached").versions.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_strict_borrowed_matches_shallow_headers_and_rejects_corrupt_or_divergent_buffers() {
|
||||
let base_bytes = create_real_xlmeta().expect("create real xlmeta");
|
||||
let mut peer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
|
||||
// Mutate inner version payload (ver.meta) without changing shallow header.
|
||||
let mut ver = FileMetaVersion::try_from(peer_fm.versions[1].meta.as_slice()).expect("decode version");
|
||||
ver.object.as_mut().expect("object version").erasure_index += 1;
|
||||
peer_fm.versions[1].meta = ver.marshal_msg().expect("marshal peer version");
|
||||
let peer_bytes = peer_fm.marshal_msg().expect("marshal peer xlmeta");
|
||||
assert_ne!(base_bytes, peer_bytes, "peer shard has distinct erasure index payload");
|
||||
|
||||
let mut e1 = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: base_bytes.clone(),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let e1_clone = e1.clone();
|
||||
let e2 = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: peer_bytes,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
assert!(e1.matches_strict_borrowed(&e2));
|
||||
assert!(e1.matches_strict_borrowed(&e1_clone));
|
||||
|
||||
// Different shallow header must not match.
|
||||
let mut newer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
|
||||
newer_fm.versions[0].header.version_id = Some(Uuid::new_v4());
|
||||
let newer_entry = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: newer_fm.marshal_msg().expect("marshal newer xlmeta"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
assert!(!e1.matches_strict_borrowed(&newer_entry));
|
||||
|
||||
// Identical corrupt buffers must return false, never true via byte-equality shortcut.
|
||||
let mut corrupt = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: b"XL2 corrupt-payload".to_vec(),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let corrupt_peer = corrupt.clone();
|
||||
assert!(!corrupt.matches_strict_borrowed(&corrupt_peer));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_peek_buffered_and_skip_current_match_async_peek_and_skip() {
|
||||
let entries: Vec<_> = (0..5)
|
||||
.map(|i| MetaCacheEntry {
|
||||
name: format!("buffered-key-{i}"),
|
||||
metadata: vec![i as u8; 16],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})
|
||||
.collect();
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut buf);
|
||||
writer.write(&entries).await.expect("write entries");
|
||||
writer.close_with_limit_reached(false).await.expect("close stream");
|
||||
|
||||
let mut reader = MetacacheReader::new(Cursor::new(buf.into_inner()));
|
||||
assert!(reader.try_peek_buffered().is_none(), "uninitialized reader must defer to async peek");
|
||||
assert!(!reader.skip_current(), "no current entry before first peek");
|
||||
|
||||
let first = reader.peek().await.expect("first peek").expect("entry 0");
|
||||
assert_eq!(first, entries[0]);
|
||||
assert!(reader.skip_current(), "skip_current must consume the peeked entry");
|
||||
assert!(!reader.skip_current(), "second skip_current without peek must return false");
|
||||
|
||||
for expected in &entries[1..] {
|
||||
let next = reader
|
||||
.try_peek_buffered()
|
||||
.expect("remaining entries are already buffered")
|
||||
.expect("valid entry")
|
||||
.expect("present entry");
|
||||
assert_eq!(&next, expected);
|
||||
assert!(reader.skip_current());
|
||||
}
|
||||
|
||||
assert!(
|
||||
reader.try_peek_buffered().is_none(),
|
||||
"stream terminator must defer to async peek for trailer handling"
|
||||
);
|
||||
assert!(reader.peek().await.expect("eof peek").is_none());
|
||||
assert_eq!(reader.limit_reached(), Some(false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
@@ -1550,6 +1551,7 @@ impl ErasureSetHealer {
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
let page_resume_index = *current_object_index;
|
||||
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
|
||||
let mut object_locks: HashMap<String, Arc<tokio::sync::Mutex<()>>> = HashMap::new();
|
||||
let mut page_tasks = FuturesUnordered::new();
|
||||
let mut completed_in_page = 0usize;
|
||||
|
||||
@@ -1721,6 +1723,7 @@ impl ErasureSetHealer {
|
||||
let set_label = set_disk_id.to_string();
|
||||
let heal_opts = self.heal_opts;
|
||||
let semaphore = semaphore.clone();
|
||||
let object_lock = object_locks.entry(object_name.clone()).or_default().clone();
|
||||
let target_endpoints = self.target_endpoints.clone();
|
||||
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
|
||||
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
|
||||
@@ -1735,6 +1738,13 @@ impl ErasureSetHealer {
|
||||
.as_ref()
|
||||
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
|
||||
let work = async move {
|
||||
// Versions share one namespace lock. Queue them before taking a
|
||||
// page permit so a waiting version cannot block other objects.
|
||||
let _object_guard = tokio::select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
|
||||
guard = object_lock.lock() => guard,
|
||||
};
|
||||
let mut retries = 0;
|
||||
let mut admin_identity = admin_identity;
|
||||
let mut admin_task_id = admin_task_id;
|
||||
@@ -4047,6 +4057,64 @@ mod resume_loop_tests {
|
||||
assert_eq!(unique.len(), 50, "no version healed twice");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn page_serializes_versions_without_blocking_other_objects() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
.with_replacement_targets(vec!["replacement-a".into()], Some(env.task_id.clone()));
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
*env.storage.heal_gate.lock().unwrap() = Some(gate.clone());
|
||||
for (name, version) in [("a", "v1"), ("a", "v2"), ("b", "v1")] {
|
||||
env.storage
|
||||
.set_result(name, Some(version), replacement_target_ok_result("replacement-a", name));
|
||||
}
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![
|
||||
item("a", Some("v1"), false),
|
||||
item("a", Some("v2"), false),
|
||||
item("b", Some("v1"), false),
|
||||
],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
let observe = async {
|
||||
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
let notified = env.storage.heal_started.notified();
|
||||
if env.storage.calls().len() >= 2 {
|
||||
break;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok();
|
||||
let calls = env.storage.calls();
|
||||
gate.add_permits(3);
|
||||
assert!(started, "different objects should run concurrently");
|
||||
assert_eq!(calls.len(), 2);
|
||||
assert!(calls.iter().any(|(name, _)| name == "a"));
|
||||
assert!(calls.iter().any(|(name, _)| name == "b"));
|
||||
};
|
||||
let ((processed, successful, failed, skipped, result), ()) = tokio::join!(run(&env), observe);
|
||||
result.expect("all versions heal");
|
||||
assert_eq!((processed, successful, failed, skipped), (3, 3, 0, 0));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_replacement_runs_two_object_repairs_within_the_configured_limit() {
|
||||
temp_env::async_with_vars(
|
||||
|
||||
@@ -420,6 +420,7 @@ async fn real_storage_grace_round_trip() {
|
||||
&HealOpts {
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
recreate: true,
|
||||
remove: true,
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
|
||||
@@ -14,13 +14,26 @@
|
||||
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
|
||||
use super::*;
|
||||
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
|
||||
use crate::heal::storage::HealListItem;
|
||||
use crate::heal::storage::{HealListItem, HealStorageObjectResult};
|
||||
use crate::heal::utils::format_set_disk_id;
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use std::collections::HashSet;
|
||||
|
||||
const MAX_DEFERRED_OBJECTS: usize = 256;
|
||||
const MAX_DEFERRED_BYTES: usize = 256 * 1024;
|
||||
const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
|
||||
const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
|
||||
const MAX_INFLIGHT_REPAIRS: usize = 8;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn deferred_payload_budget_for_test() -> usize {
|
||||
MAX_DEFERRED_BYTES - MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>()
|
||||
}
|
||||
// A recovered pool can become writable just after the ordinary retry budget.
|
||||
// Keep this extra attempt inside the existing deferred age and backoff caps.
|
||||
const MAX_SLOWDOWN_OBJECT_RETRIES: u32 = MAX_BUCKET_OBJECT_HEAL_RETRIES + 1;
|
||||
|
||||
type HealListingPage = (Vec<HealListItem>, Option<String>, bool);
|
||||
|
||||
struct DeferredObject {
|
||||
name: String,
|
||||
@@ -57,7 +70,11 @@ impl DeferredObject {
|
||||
let now = tokio::time::Instant::now();
|
||||
let first = *self.first_failure.get_or_insert(now);
|
||||
self.attempt += 1;
|
||||
self.due = (now + delay).min(first + MAX_DEFERRED_AGE);
|
||||
let remaining = (first + MAX_DEFERRED_AGE).saturating_duration_since(now);
|
||||
// Scheduling exactly at the deadline guarantees the retry will be
|
||||
// rejected as expired without making the storage attempt.
|
||||
let delay = if delay >= remaining { remaining / 2 } else { delay };
|
||||
self.due = now + delay;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,16 +106,23 @@ impl DeferredWindow {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pop_due(&mut self) -> Option<DeferredObject> {
|
||||
fn pop_due(&mut self, in_flight: &HashSet<String>) -> Option<DeferredObject> {
|
||||
let now = tokio::time::Instant::now();
|
||||
let index = self.objects.iter().position(|item| item.due <= now)?;
|
||||
let index = self
|
||||
.objects
|
||||
.iter()
|
||||
.position(|item| item.due <= now && !in_flight.contains(&item.name))?;
|
||||
let item = self.objects.remove(index)?;
|
||||
self.bytes -= item.payload_bytes();
|
||||
Some(item)
|
||||
}
|
||||
|
||||
fn next_due(&self) -> Option<tokio::time::Instant> {
|
||||
self.objects.iter().map(|item| item.due).min()
|
||||
fn next_due(&self, in_flight: &HashSet<String>) -> Option<tokio::time::Instant> {
|
||||
self.objects
|
||||
.iter()
|
||||
.filter(|item| !in_flight.contains(&item.name))
|
||||
.map(|item| item.due)
|
||||
.min()
|
||||
}
|
||||
|
||||
fn can_advance(&self, page: u64) -> bool {
|
||||
@@ -132,6 +156,32 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
async fn heal_bucket_storage_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
item: DeferredObject,
|
||||
incarnation: Option<Uuid>,
|
||||
opts: &HealOpts,
|
||||
) -> (DeferredObject, Result<HealStorageObjectResult>) {
|
||||
let result = self
|
||||
.await_with_control(async {
|
||||
match incarnation {
|
||||
Some(expected) => {
|
||||
self.storage
|
||||
.heal_object_at_incarnation(bucket, &item.name, item.version_id.as_deref(), expected, opts)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.storage
|
||||
.heal_object_with_receipt(bucket, &item.name, item.version_id.as_deref(), opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
(item, result)
|
||||
}
|
||||
|
||||
async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result<bool> {
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
@@ -649,56 +699,168 @@ impl HealTask {
|
||||
};
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut deferred = DeferredWindow::default();
|
||||
let mut inline_retry: Option<DeferredObject> = None;
|
||||
// Only already-admitted operations can overflow the deferred byte
|
||||
// budget. Stop listing and page advancement while these drain.
|
||||
let mut inline_retry = VecDeque::<DeferredObject>::new();
|
||||
let mut page_number = 0_u64;
|
||||
let mut aborted_progress_unknown = false;
|
||||
let mut pending = Vec::<HealListItem>::new().into_iter();
|
||||
let mut listing_finished = false;
|
||||
let mut listing_attempt = 0;
|
||||
let mut listing_due = tokio::time::Instant::now();
|
||||
let mut listing_inflight: Option<futures::future::BoxFuture<'static, Result<HealListingPage>>> = None;
|
||||
let mut in_flight = HashSet::new();
|
||||
let repairs_aborted = Arc::new(AtomicBool::new(false));
|
||||
let mut repairs: FuturesUnordered<
|
||||
futures::future::BoxFuture<'_, (DeferredObject, Result<HealStorageObjectResult>, bool)>,
|
||||
> = FuturesUnordered::new();
|
||||
let scope_result: Result<()> = async {
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
if listing_finished && pending.as_slice().is_empty() && deferred.objects.is_empty() && inline_retry.is_none()
|
||||
if listing_finished
|
||||
&& pending.as_slice().is_empty()
|
||||
&& deferred.objects.is_empty()
|
||||
&& inline_retry.is_empty()
|
||||
&& repairs.is_empty()
|
||||
{
|
||||
break;
|
||||
}
|
||||
self.pace_mainline().await?;
|
||||
// Listing and object retries share this safe boundary. A
|
||||
// failed listing never hides an already-due object retry.
|
||||
let item = deferred.pop_due().or_else(|| {
|
||||
if inline_retry
|
||||
.as_ref()
|
||||
.is_some_and(|item| item.due <= tokio::time::Instant::now())
|
||||
{
|
||||
inline_retry.take()
|
||||
} else if inline_retry.is_none() {
|
||||
let item = if repairs.len() < MAX_INFLIGHT_REPAIRS {
|
||||
let due = inline_retry
|
||||
.iter()
|
||||
.position(|item| item.due <= tokio::time::Instant::now() && !in_flight.contains(&item.name));
|
||||
due.and_then(|index| inline_retry.remove(index))
|
||||
.or_else(|| deferred.pop_due(&in_flight))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
.or_else(|| {
|
||||
if inline_retry.is_empty() && repairs.is_empty() {
|
||||
pending.next().map(|item| DeferredObject::new(item, page_number))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
let Some(mut item) = item else {
|
||||
let can_list = !listing_finished && inline_retry.is_none() && deferred.can_advance(page_number);
|
||||
let (mut item, storage_result, age_exhausted) = if let Some(item) = item {
|
||||
if item.expired() {
|
||||
(item, Err(Error::other("heal object retry age exhausted")), true)
|
||||
} else {
|
||||
in_flight.insert(item.name.clone());
|
||||
let opts = &heal_opts;
|
||||
let aborted = Arc::clone(&repairs_aborted);
|
||||
repairs.push(Box::pin(async move {
|
||||
if aborted.load(Ordering::Acquire) {
|
||||
return (item, Err(Error::TaskCancelled), false);
|
||||
}
|
||||
if let Err(error) = self.pace_mainline().await {
|
||||
return (item, Err(error), false);
|
||||
}
|
||||
if aborted.load(Ordering::Acquire) {
|
||||
return (item, Err(Error::TaskCancelled), false);
|
||||
}
|
||||
let (item, result) = self
|
||||
.heal_bucket_storage_object(bucket, item, traversal_incarnation_id, opts)
|
||||
.await;
|
||||
(item, result, true)
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
} else if !repairs.is_empty() {
|
||||
let next_retry = if repairs.len() < MAX_INFLIGHT_REPAIRS {
|
||||
deferred
|
||||
.next_due(&in_flight)
|
||||
.into_iter()
|
||||
.chain(
|
||||
inline_retry
|
||||
.iter()
|
||||
.filter(|item| !in_flight.contains(&item.name))
|
||||
.map(|item| item.due),
|
||||
)
|
||||
.min()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed = if let Some(due) = next_retry {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = tokio::time::sleep_until(due) => None,
|
||||
completed = repairs.next() => completed,
|
||||
}
|
||||
} else {
|
||||
repairs.next().await
|
||||
};
|
||||
let Some((item, result, attempted)) = completed else {
|
||||
continue;
|
||||
};
|
||||
in_flight.remove(&item.name);
|
||||
if !attempted && item.attempt == 0 {
|
||||
return Err(result.expect_err("an unstarted repair must have a control error"));
|
||||
}
|
||||
(item, result, false)
|
||||
} else {
|
||||
self.pace_mainline().await?;
|
||||
let can_list = !listing_finished && inline_retry.is_empty() && deferred.can_advance(page_number);
|
||||
if can_list && listing_due <= tokio::time::Instant::now() {
|
||||
let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
|
||||
self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
|
||||
set_disk_id,
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_objects_for_heal_page(
|
||||
bucket,
|
||||
prefix,
|
||||
continuation_token.as_deref(),
|
||||
false,
|
||||
))
|
||||
.await
|
||||
if listing_inflight.is_none() {
|
||||
let storage = Arc::clone(&self.storage);
|
||||
let listing_set = set_disk_id.clone();
|
||||
let listing_bucket = bucket.to_owned();
|
||||
let listing_prefix = prefix.to_owned();
|
||||
let cursor = continuation_token.clone();
|
||||
listing_inflight = Some(Box::pin(async move {
|
||||
if let Some(set_id) = listing_set.as_deref() {
|
||||
storage
|
||||
.list_versions_for_heal_page_disk_walk(
|
||||
set_id,
|
||||
&listing_bucket,
|
||||
&listing_prefix,
|
||||
cursor.as_deref(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
storage
|
||||
.list_objects_for_heal_page(
|
||||
&listing_bucket,
|
||||
&listing_prefix,
|
||||
cursor.as_deref(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}));
|
||||
}
|
||||
let next_retry = deferred
|
||||
.next_due(&in_flight)
|
||||
.into_iter()
|
||||
.chain(inline_retry.iter().map(|item| item.due))
|
||||
.min();
|
||||
let Some(listing) = listing_inflight.as_mut() else {
|
||||
return Err(Error::other("heal listing future is unavailable"));
|
||||
};
|
||||
// Keep the same listing future alive while an earlier retry runs.
|
||||
// Restarting a disk walk here would duplicate work and may change its snapshot.
|
||||
let result = self
|
||||
.await_with_control(async {
|
||||
if let Some(due) = next_retry {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = tokio::time::sleep_until(due) => Ok(None),
|
||||
result = listing => Ok(Some(result)),
|
||||
}
|
||||
} else {
|
||||
Ok(Some(listing.await))
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let page = match result {
|
||||
Ok(Some(page)) => page,
|
||||
Ok(None) => continue,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
listing_inflight = None;
|
||||
match page {
|
||||
Ok((objects, next_token, is_truncated)) => {
|
||||
page_number = page_number.saturating_add(1);
|
||||
@@ -727,9 +889,9 @@ impl HealTask {
|
||||
continue;
|
||||
} else {
|
||||
let due = deferred
|
||||
.next_due()
|
||||
.next_due(&in_flight)
|
||||
.into_iter()
|
||||
.chain(inline_retry.as_ref().map(|item| item.due))
|
||||
.chain(inline_retry.iter().map(|item| item.due))
|
||||
.chain(can_list.then_some(listing_due))
|
||||
.min();
|
||||
if let Some(due) = due {
|
||||
@@ -761,89 +923,74 @@ impl HealTask {
|
||||
}
|
||||
|
||||
let mut terminal_outcome = true;
|
||||
let age_exhausted = item.expired();
|
||||
let error = if age_exhausted {
|
||||
Some(Error::other("heal object retry age exhausted"))
|
||||
} else {
|
||||
match self
|
||||
.await_with_control(async {
|
||||
match traversal_incarnation_id {
|
||||
Some(expected) => {
|
||||
self.storage
|
||||
.heal_object_at_incarnation(
|
||||
bucket,
|
||||
object,
|
||||
item.version_id.as_deref(),
|
||||
expected,
|
||||
&heal_opts,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.storage
|
||||
.heal_object_with_receipt(bucket, object, item.version_id.as_deref(), &heal_opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(storage_result) if storage_result.error.is_none() => {
|
||||
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
#[cfg(test)]
|
||||
super::pause_storage_receipt(&self.id).await;
|
||||
recorded_authoritative_outcome =
|
||||
self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
|
||||
if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
|
||||
Some(Error::TaskCancelled)
|
||||
} else {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |= !add_bytes(
|
||||
&mut bytes,
|
||||
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||
);
|
||||
if !recorded_authoritative_outcome
|
||||
&& !heal_opts.dry_run
|
||||
&& !storage_result.item.detail.is_empty()
|
||||
{
|
||||
detail = Some(storage_result.item.detail.clone());
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
None
|
||||
let object = item.name.as_str();
|
||||
let error = match storage_result {
|
||||
Ok(storage_result) if storage_result.error.is_none() => {
|
||||
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
#[cfg(test)]
|
||||
super::pause_storage_receipt(&self.id).await;
|
||||
recorded_authoritative_outcome =
|
||||
self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
|
||||
if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
|
||||
Some(Error::TaskCancelled)
|
||||
} else {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |= !add_bytes(
|
||||
&mut bytes,
|
||||
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||
);
|
||||
if !recorded_authoritative_outcome
|
||||
&& !heal_opts.dry_run
|
||||
&& !storage_result.item.detail.is_empty()
|
||||
{
|
||||
detail = Some(storage_result.item.detail.clone());
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(storage_result)
|
||||
if storage_result
|
||||
.error
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
|
||||
{
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "object_dir_not_found_skipped",
|
||||
"Heal bucket object-dir candidate skipped after not-found result"
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok(storage_result) => storage_result.error,
|
||||
Err(err) => Some(err),
|
||||
}
|
||||
Ok(storage_result)
|
||||
if storage_result
|
||||
.error
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
|
||||
{
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "object_dir_not_found_skipped",
|
||||
"Heal bucket object-dir candidate skipped after not-found result"
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok(storage_result) => storage_result.error,
|
||||
Err(err) => Some(err),
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
match err {
|
||||
Error::StaleBucketIncarnation { .. } => return Err(err),
|
||||
Error::StaleBucketIncarnation { .. } => {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity,
|
||||
disposition: HealObjectDisposition::Unknown,
|
||||
detail: None,
|
||||
});
|
||||
aborted_progress_unknown |= !increment_counter(&mut scanned);
|
||||
aborted_progress_unknown |= !increment_counter(&mut skipped);
|
||||
return Err(err);
|
||||
}
|
||||
Error::TaskCancelled | Error::TaskTimeout => {
|
||||
let disposition = if matches!(err, Error::TaskCancelled) {
|
||||
HealObjectDisposition::Cancelled
|
||||
@@ -912,7 +1059,12 @@ impl HealTask {
|
||||
);
|
||||
} else if !age_exhausted
|
||||
&& err.is_recoverable_object_heal()
|
||||
&& retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES
|
||||
&& retry_attempt
|
||||
< if matches!(&err, Error::Storage(crate::heal::StorageError::SlowDown)) {
|
||||
MAX_SLOWDOWN_OBJECT_RETRIES
|
||||
} else {
|
||||
MAX_BUCKET_OBJECT_HEAL_RETRIES
|
||||
}
|
||||
{
|
||||
terminal_outcome = false;
|
||||
debug!(
|
||||
@@ -930,7 +1082,7 @@ impl HealTask {
|
||||
);
|
||||
item.defer(Self::bucket_object_retry_delay(&self.id, retry_attempt + 1));
|
||||
if let Err(item) = deferred.push(item) {
|
||||
inline_retry = Some(item);
|
||||
inline_retry.push_back(item);
|
||||
}
|
||||
} else {
|
||||
disposition = HealObjectDisposition::Failed(if age_exhausted || err.is_recoverable_object_heal() {
|
||||
@@ -947,17 +1099,24 @@ impl HealTask {
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
let retry_age_ms = item
|
||||
.first_failure
|
||||
.map(|first| u64::try_from(first.elapsed().as_millis()).unwrap_or(u64::MAX));
|
||||
let retry_due_lag_ms =
|
||||
age_exhausted.then(|| u64::try_from(item.due.elapsed().as_millis()).unwrap_or(u64::MAX));
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
result = "object_failed",
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
retry_age_ms = ?retry_age_ms,
|
||||
retry_due_lag_ms = ?retry_due_lag_ms,
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
);
|
||||
}
|
||||
@@ -998,6 +1157,7 @@ impl HealTask {
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = scope_result {
|
||||
repairs_aborted.store(true, Ordering::Release);
|
||||
let disposition = match error {
|
||||
Error::TaskCancelled => HealObjectDisposition::Cancelled,
|
||||
Error::TaskTimeout => HealObjectDisposition::Deferred {
|
||||
@@ -1006,20 +1166,51 @@ impl HealTask {
|
||||
},
|
||||
_ => HealObjectDisposition::Unknown,
|
||||
};
|
||||
// Keep issued replacement I/O alive until it reaches its storage
|
||||
// boundary, even when another operation aborts this traversal.
|
||||
while let Some((item, result, attempted)) = repairs.next().await {
|
||||
if !attempted && item.attempt == 0 {
|
||||
continue;
|
||||
}
|
||||
let mut identity =
|
||||
self.outcome_identity(bucket, &item.name, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
identity.bucket_incarnation_id = bucket_incarnation_id;
|
||||
let mut recorded = false;
|
||||
if let Ok(storage_result) = result
|
||||
&& storage_result.error.is_none()
|
||||
&& unavailable_recreate_error(&storage_result.item, &heal_opts).is_none()
|
||||
{
|
||||
recorded = self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
|
||||
if recorded {
|
||||
aborted_progress_unknown |= !increment_counter(&mut healed);
|
||||
aborted_progress_unknown |=
|
||||
!add_bytes(&mut bytes, u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX));
|
||||
self.record_result_item(storage_result.item).await;
|
||||
}
|
||||
}
|
||||
if !recorded {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity,
|
||||
disposition: disposition.clone(),
|
||||
detail: None,
|
||||
});
|
||||
aborted_progress_unknown |= !increment_counter(&mut skipped);
|
||||
}
|
||||
aborted_progress_unknown |= !increment_counter(&mut scanned);
|
||||
}
|
||||
// Only attempted identities have terminal outcomes. Unstarted
|
||||
// page tails remain unprocessed under the task's partial coverage.
|
||||
// No detached sleepers survive abort.
|
||||
for item in deferred.objects.into_iter().chain(inline_retry) {
|
||||
let mut identity =
|
||||
self.outcome_identity(bucket, &item.name, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
identity.bucket_incarnation_id = bucket_incarnation_id;
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: self.outcome_identity(
|
||||
bucket,
|
||||
&item.name,
|
||||
item.version_id.as_deref(),
|
||||
heal_opts.pool,
|
||||
heal_opts.set,
|
||||
),
|
||||
identity,
|
||||
disposition: disposition.clone(),
|
||||
detail: None,
|
||||
});
|
||||
|
||||
@@ -1658,7 +1658,9 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
struct MockStorage {
|
||||
retry_test_pages: Option<Vec<Vec<HealListItem>>>,
|
||||
retry_test_delays: HashMap<String, Duration>,
|
||||
retry_test_attempt_delays: Mutex<HashMap<String, VecDeque<Duration>>>,
|
||||
retry_test_listing_delays: Mutex<VecDeque<Duration>>,
|
||||
retry_test_disk_walk_pages: bool,
|
||||
retry_test_events: Mutex<Vec<String>>,
|
||||
listed: Mutex<bool>,
|
||||
list_each_bucket: bool,
|
||||
@@ -1678,6 +1680,7 @@ struct MockStorage {
|
||||
object_heal_detail: Option<String>,
|
||||
object_heal_item: Option<HealResultItem>,
|
||||
bucket_incarnation_id: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_reads: std::sync::atomic::AtomicUsize,
|
||||
bucket_incarnation_after_object_heal: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_after_listing: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_unavailable: Mutex<bool>,
|
||||
@@ -2427,6 +2430,7 @@ impl HealStorageAPI for MockStorage {
|
||||
}
|
||||
|
||||
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.bucket_incarnation_reads.fetch_add(1, Ordering::Relaxed);
|
||||
if *self.bucket_incarnation_unavailable.lock().unwrap() {
|
||||
return Err(Error::Other("bucket incarnation unavailable".to_string()));
|
||||
}
|
||||
@@ -2459,8 +2463,15 @@ impl HealStorageAPI for MockStorage {
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.heal_object_calls.lock().unwrap().push(object.to_string());
|
||||
self.retry_test_events.lock().expect("events").push(format!("heal:{object}"));
|
||||
if let Some(delay) = self.retry_test_delays.get(object) {
|
||||
tokio::time::sleep(*delay).await;
|
||||
let delay = self
|
||||
.retry_test_attempt_delays
|
||||
.lock()
|
||||
.expect("attempt delays")
|
||||
.get_mut(object)
|
||||
.and_then(VecDeque::pop_front)
|
||||
.or_else(|| self.retry_test_delays.get(object).copied());
|
||||
if let Some(delay) = delay {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
self.heal_object_version_ids
|
||||
.lock()
|
||||
@@ -2913,6 +2924,11 @@ impl HealStorageAPI for MockStorage {
|
||||
_include_lifecycle_object_info: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
self.disk_walk_calls.lock().unwrap().push(set_disk_id.to_string());
|
||||
if self.retry_test_disk_walk_pages {
|
||||
return self
|
||||
.list_objects_for_heal_page(_bucket, _prefix, continuation_token, _include_lifecycle_object_info)
|
||||
.await;
|
||||
}
|
||||
if continuation_token.is_some() {
|
||||
return Ok((Vec::new(), None, false));
|
||||
}
|
||||
|
||||
@@ -52,7 +52,92 @@ fn fail_once(storage: &MockStorage, name: &str) {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
|
||||
async fn slow_listing_does_not_starve_due_object_retry() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "heal:object-a"],
|
||||
"the object retry must run while the slow listing remains in flight"
|
||||
);
|
||||
assert_eq!(task.get_outcome().await.counters.failed, 0);
|
||||
execution.await.expect("the resumed listing and retried object complete");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
[
|
||||
"list:first",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-b"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_set_disk_walk_does_not_starve_due_object_retry() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
erasure_set_scopes: Mutex::new(vec![(0, 0)]),
|
||||
retry_test_disk_walk_pages: true,
|
||||
retry_test_pages: Some(vec![vec![heal_item("a")], vec![heal_item("b")]]),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:a", "list:1", "heal:a"]
|
||||
);
|
||||
assert_eq!(storage.disk_walk_calls.lock().expect("walks").len(), 2);
|
||||
execution.await.expect("the same disk walk resumes and both objects heal");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "a", "b"]);
|
||||
assert_eq!(storage.disk_walk_calls.lock().expect("walks").len(), 2);
|
||||
assert_eq!(task.get_outcome().await.counters.failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancelling_an_inflight_listing_keeps_deferred_outcome_cancelled() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second"]
|
||||
);
|
||||
task.cancel().await.expect("cancel");
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
|
||||
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_listing_services_multiple_retries_without_restarting_walk() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(29)])),
|
||||
@@ -68,22 +153,17 @@ async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(30_500), &mut execution)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "heal:object-a"]
|
||||
["list:first", "heal:object-a", "list:second", "heal:object-a", "heal:object-a"]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(1, 0, 2)
|
||||
);
|
||||
execution.await.expect_err("age exhausted object must remain a batch failure");
|
||||
execution.await.expect("object and resumed listing complete");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
[
|
||||
@@ -91,22 +171,25 @@ async fn slow_listing_retry_services_due_object_then_age_before_next_listing() {
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-a",
|
||||
"heal:object-a",
|
||||
"list:second",
|
||||
"heal:object-b"
|
||||
]
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.attempt_failures), (2, 3));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn listing_return_after_age_expires_does_not_start_another_heal_attempt() {
|
||||
async fn slow_listing_preserves_retry_limit_and_failure() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
retry_test_listing_delays: Mutex::new(VecDeque::from([Duration::ZERO, Duration::from_secs(31)])),
|
||||
..Default::default()
|
||||
});
|
||||
fail_once(&storage, "object-a");
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("object-a".to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
@@ -115,15 +198,20 @@ async fn listing_return_after_age_expires_does_not_start_another_heal_attempt()
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["object-a"; 5]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second"]
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(1, 1, 6) // Five object failures and one listing failure.
|
||||
);
|
||||
assert_eq!(task.get_outcome().await.counters.failed, 1);
|
||||
execution.await.expect_err("age exhaustion remains a failure");
|
||||
assert_eq!(
|
||||
storage.retry_test_events.lock().expect("events").as_slice(),
|
||||
["list:first", "heal:object-a", "list:second", "list:second", "heal:object-b"]
|
||||
outcome.objects[0].disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
);
|
||||
execution.await.expect_err("exhausted object must remain a batch failure");
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").last().map(String::as_str),
|
||||
Some("object-b")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -220,6 +308,29 @@ async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unproc
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slowdown_can_recover_within_existing_deferred_age_budget() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"]]));
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("a".to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
let task = bucket_task(storage.clone());
|
||||
let started = tokio::time::Instant::now();
|
||||
task.execute()
|
||||
.await
|
||||
.expect("transient slowdown recovers within the deferred age budget");
|
||||
assert!(started.elapsed() < Duration::from_secs(30));
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 5);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(1, 0, 4)
|
||||
);
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
|
||||
@@ -228,7 +339,7 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
.insert(name.to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
}
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
@@ -239,9 +350,9 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(4, 4, 16)
|
||||
(4, 4, 20)
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 20);
|
||||
for name in ["a", "b", "c", "d"] {
|
||||
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
|
||||
}
|
||||
@@ -294,14 +405,360 @@ async fn due_retry_runs_before_next_object_in_a_slow_healthy_page() {
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
|
||||
let mut storage = pages_storage(&[&["a"], &["b"]]);
|
||||
async fn slow_unrelated_object_does_not_starve_due_retry() {
|
||||
let mut storage = pages_storage(&[&["a", "b"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("aged pending responsibility is not success");
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["a", "b", "a"],
|
||||
"a due retry must run while an unrelated object is still healing"
|
||||
);
|
||||
execution.await.expect("both objects complete");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.failed), (2, 0));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_drains_slow_replacement_object_after_a_concurrent_retry() {
|
||||
let mut storage = pages_storage(&[&["a", "b"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
storage
|
||||
.retry_test_attempt_delays
|
||||
.lock()
|
||||
.expect("attempt delays")
|
||||
.insert("a".to_string(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)]));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let owner = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
|
||||
*task.replacement_execution.write().await = Some(owner);
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a"]);
|
||||
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 1);
|
||||
task.cancel_token.cancel();
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
assert!(task.replacement_execution.read().await.is_none());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a"]);
|
||||
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stale_bucket_retry_drains_another_inflight_replacement() {
|
||||
let original = Uuid::new_v4();
|
||||
let mut storage = pages_storage(&[&["a", "b"]]);
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let mut task = bucket_task(storage.clone());
|
||||
task.bucket_incarnation_id = Some(original);
|
||||
let owner = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
|
||||
*task.replacement_execution.write().await = Some(owner);
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b"]);
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4());
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
let result = execution.await;
|
||||
assert!(
|
||||
matches!(result, Err(Error::StaleBucketIncarnation { .. })),
|
||||
"unexpected result: {result:?}"
|
||||
);
|
||||
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 2);
|
||||
assert!(task.replacement_execution.read().await.is_none());
|
||||
let outcome = task.get_outcome().await;
|
||||
let stale = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("stale retry outcome");
|
||||
assert_eq!(stale.identity.bucket_incarnation_id, Some(original));
|
||||
assert_eq!(stale.disposition, HealObjectDisposition::Unknown);
|
||||
let other = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "b")
|
||||
.expect("b outcome");
|
||||
assert_eq!(other.identity.bucket_incarnation_id, Some(original));
|
||||
assert_eq!(other.disposition, HealObjectDisposition::Unknown);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stale_retry_preserves_verified_receipt_from_another_inflight_replacement() {
|
||||
let original = Uuid::new_v4();
|
||||
let mut storage = pages_storage(&[&["a", "b"]]);
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
storage.heal_object_receipts.lock().expect("receipts").insert(
|
||||
"b".to_string(),
|
||||
VecDeque::from([object_receipt("b", None, HealObjectDisposition::Repaired, original)]),
|
||||
);
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let mut task = bucket_task(storage.clone());
|
||||
task.bucket_incarnation_id = Some(original);
|
||||
*task.replacement_execution.write().await =
|
||||
Some(crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new()));
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4());
|
||||
assert!(matches!(execution.await, Err(Error::StaleBucketIncarnation { .. })));
|
||||
let outcome = task.get_outcome().await;
|
||||
let repaired = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "b")
|
||||
.expect("b outcome");
|
||||
assert_eq!(repaired.identity.bucket_incarnation_id, Some(original));
|
||||
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stale_retry_does_not_start_other_queued_retries_while_draining() {
|
||||
let original = Uuid::new_v4();
|
||||
let mut storage = pages_storage(&[&["a", "c", "b"]]);
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(original);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
fail_once(&storage, "c");
|
||||
let storage = Arc::new(storage);
|
||||
let mut task = bucket_task(storage.clone());
|
||||
task.bucket_incarnation_id = Some(original);
|
||||
*task.replacement_execution.write().await =
|
||||
Some(crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new()));
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "c", "b"]);
|
||||
let before = storage.bucket_incarnation_reads.load(Ordering::Relaxed);
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(Uuid::new_v4());
|
||||
assert!(matches!(execution.await, Err(Error::StaleBucketIncarnation { .. })));
|
||||
assert_eq!(storage.bucket_incarnation_reads.load(Ordering::Relaxed), before + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_retries_keep_inflight_work_bounded() {
|
||||
let names: Vec<String> = (0..10).map(|index| format!("retry-{index}")).collect();
|
||||
let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect();
|
||||
page.push(heal_item("slow"));
|
||||
let mut storage = MockStorage {
|
||||
retry_test_pages: Some(vec![page]),
|
||||
..Default::default()
|
||||
};
|
||||
storage.retry_test_delays.insert("slow".to_string(), Duration::from_secs(31));
|
||||
for name in &names {
|
||||
fail_once(&storage, name);
|
||||
storage
|
||||
.retry_test_attempt_delays
|
||||
.lock()
|
||||
.expect("attempt delays")
|
||||
.insert(name.clone(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)]));
|
||||
}
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 18);
|
||||
assert_eq!(storage.heal_object_version_ids.lock().expect("completed calls").len(), 10);
|
||||
task.cancel_token.cancel();
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 18);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn inline_retry_is_not_starved_by_slow_deferred_retries() {
|
||||
let names: Vec<_> = (0..256).map(|index| format!("slow-{index:03}")).collect();
|
||||
let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect();
|
||||
page.push(heal_item("inline"));
|
||||
let storage = MockStorage {
|
||||
retry_test_pages: Some(vec![page]),
|
||||
..Default::default()
|
||||
};
|
||||
for name in &names {
|
||||
fail_once(&storage, name);
|
||||
storage
|
||||
.retry_test_attempt_delays
|
||||
.lock()
|
||||
.expect("delays")
|
||||
.insert(name.clone(), VecDeque::from([Duration::ZERO, Duration::from_secs(31)]));
|
||||
}
|
||||
fail_once(&storage, "inline");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(10), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage
|
||||
.heal_object_calls
|
||||
.lock()
|
||||
.expect("calls")
|
||||
.iter()
|
||||
.filter(|name| name.as_str() == "inline")
|
||||
.count(),
|
||||
2,
|
||||
"the overflow identity must be retried before slow work occupies every slot"
|
||||
);
|
||||
task.cancel_token.cancel();
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn concurrent_retry_overflow_preserves_every_attempted_identity() {
|
||||
assert_concurrent_retry_overflow_preserves_identities(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn concurrent_retry_overflow_completes_every_attempted_identity() {
|
||||
assert_concurrent_retry_overflow_preserves_identities(false).await;
|
||||
}
|
||||
|
||||
async fn assert_concurrent_retry_overflow_preserves_identities(cancel: bool) {
|
||||
let budget = super::super::heal_bucket::deferred_payload_budget_for_test();
|
||||
let mut lengths = vec![100; 7];
|
||||
let mut remaining = budget - 300 - 700;
|
||||
while remaining > 1_000 {
|
||||
lengths.push(1_000);
|
||||
remaining -= 1_000;
|
||||
}
|
||||
if remaining > 0 && remaining < 4 {
|
||||
*lengths.last_mut().expect("previous key") -= 4 - remaining;
|
||||
remaining = 4;
|
||||
}
|
||||
if remaining != 0 {
|
||||
lengths.push(remaining);
|
||||
}
|
||||
assert!(lengths.len() < 256);
|
||||
let names: Vec<_> = lengths
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, length)| format!("{index:04}{}", "x".repeat(length - 4)))
|
||||
.collect();
|
||||
let extra = format!("extra{}", "y".repeat(895));
|
||||
let mut page: Vec<_> = names.iter().map(|name| heal_item(name)).collect();
|
||||
page.push(heal_item(&extra));
|
||||
let storage = MockStorage {
|
||||
retry_test_pages: Some(vec![page]),
|
||||
..Default::default()
|
||||
};
|
||||
for (index, name) in names.iter().enumerate() {
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
name.clone(),
|
||||
VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]),
|
||||
);
|
||||
let delays = if index < 7 {
|
||||
vec![Duration::ZERO, Duration::from_millis(1200)]
|
||||
} else if index == 7 {
|
||||
vec![Duration::from_millis(1500)]
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
storage
|
||||
.retry_test_attempt_delays
|
||||
.lock()
|
||||
.expect("delays")
|
||||
.insert(name.clone(), delays.into());
|
||||
}
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
extra.clone(),
|
||||
VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]),
|
||||
);
|
||||
storage.retry_test_attempt_delays.lock().expect("delays").insert(
|
||||
extra,
|
||||
VecDeque::from([
|
||||
Duration::from_millis(1100),
|
||||
if cancel {
|
||||
Duration::from_secs(31)
|
||||
} else {
|
||||
Duration::from_secs(2)
|
||||
},
|
||||
]),
|
||||
);
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(5), &mut execution).await.is_err());
|
||||
if cancel {
|
||||
task.cancel_token.cancel();
|
||||
assert!(matches!(execution.await, Err(Error::TaskCancelled)));
|
||||
} else {
|
||||
execution.await.expect("all attempted identities finish");
|
||||
}
|
||||
assert_eq!(
|
||||
task.get_outcome().await.counters.processed,
|
||||
u64::try_from(names.len() + 1).expect("count")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn due_retry_does_not_overlap_another_version_of_same_object() {
|
||||
let mut first = heal_item("a");
|
||||
first.version_id = Some("first".to_string());
|
||||
let mut second = heal_item("a");
|
||||
second.version_id = Some("second".to_string());
|
||||
let mut storage = MockStorage {
|
||||
retry_test_pages: Some(vec![vec![first, second]]),
|
||||
..Default::default()
|
||||
};
|
||||
storage.retry_test_delays.insert("a".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(45), &mut execution).await.is_err());
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("calls").as_slice(),
|
||||
["a", "a"],
|
||||
"a retry must not overlap an in-flight version of the same object"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_uses_remaining_time_before_age_limit() {
|
||||
let mut storage = pages_storage(&[&["a", "b"]]);
|
||||
storage.retry_test_delays.insert("b".to_string(), Duration::from_secs(27));
|
||||
storage.heal_object_outcomes.lock().expect("outcomes").insert(
|
||||
"a".to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::RetryableLock, MockHealObjectOutcome::RetryableLock]),
|
||||
);
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect("the last retry can run before the age limit");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "b", "a", "a"]);
|
||||
assert_eq!(task.get_outcome().await.counters.failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
|
||||
let mut first = heal_item("a");
|
||||
first.version_id = Some("first".to_string());
|
||||
let mut second = heal_item("a");
|
||||
second.version_id = Some("second".to_string());
|
||||
let mut storage = MockStorage {
|
||||
retry_test_pages: Some(vec![vec![first, second]]),
|
||||
..Default::default()
|
||||
};
|
||||
storage.retry_test_delays.insert("a".to_string(), Duration::from_secs(31));
|
||||
fail_once(&storage, "a");
|
||||
let storage = Arc::new(storage);
|
||||
let task = bucket_task(storage.clone());
|
||||
task.execute().await.expect_err("aged pending responsibility is not success");
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").as_slice(), ["a", "a"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.attempt_failures, 1);
|
||||
@@ -310,7 +767,7 @@ async fn expired_retry_is_terminal_without_an_extra_storage_attempt() {
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.find(|item| item.identity.version_id.as_deref() == Some("first"))
|
||||
.expect("a outcome")
|
||||
.disposition,
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
@@ -345,13 +802,26 @@ async fn cancellation_drains_owned_retries_once() {
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn deadline_drains_owned_retries_without_false_completion() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"]]));
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = pages_storage(&[&["a"], &["b"]]);
|
||||
*storage.bucket_incarnation_id.lock().expect("incarnation") = Some(incarnation);
|
||||
let storage = Arc::new(storage);
|
||||
fail_once(&storage, "a");
|
||||
let mut task = bucket_task(storage.clone());
|
||||
task.options.timeout = Some(Duration::from_secs(1));
|
||||
assert!(matches!(task.execute().await, Err(Error::TaskTimeout)));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "a")
|
||||
.expect("a")
|
||||
.identity
|
||||
.bucket_incarnation_id,
|
||||
Some(incarnation)
|
||||
);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
|
||||
@@ -41,7 +41,7 @@ async fn count_cap_and_next_item_preserve_ownership() {
|
||||
assert_eq!(window.objects.len(), MAX_DEFERRED_OBJECTS);
|
||||
assert!(window.bytes <= MAX_DEFERRED_BYTES);
|
||||
assert!(!window.can_advance(1));
|
||||
assert!(window.pop_due().is_some());
|
||||
assert!(window.pop_due(&HashSet::new()).is_some());
|
||||
assert!(window.push(rejected).is_ok());
|
||||
}
|
||||
|
||||
@@ -56,20 +56,35 @@ async fn byte_cap_counts_key_version_and_reserved_slots() {
|
||||
assert_eq!(window.bytes, MAX_DEFERRED_BYTES);
|
||||
assert!(!window.can_advance(1));
|
||||
assert!(window.push(item("x".to_string(), None)).is_err());
|
||||
assert!(window.pop_due().is_some());
|
||||
assert!(window.pop_due(&HashSet::new()).is_some());
|
||||
assert_eq!(window.bytes, MAX_DEFERRED_OBJECTS * size_of::<DeferredObject>());
|
||||
assert!(window.push(item("x".to_string(), None)).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn due_retry_excludes_inflight_keys_without_hiding_other_keys() {
|
||||
let mut window = DeferredWindow::default();
|
||||
assert!(window.push(item("same".to_owned(), Some("v1".to_owned()))).is_ok());
|
||||
assert!(window.push(item("other".to_owned(), None)).is_ok());
|
||||
let mut in_flight = HashSet::from(["same".to_owned()]);
|
||||
assert_eq!(window.pop_due(&in_flight).expect("unblocked retry").name, "other");
|
||||
assert!(window.next_due(&in_flight).is_none());
|
||||
assert!(window.pop_due(&in_flight).is_none());
|
||||
in_flight.clear();
|
||||
assert!(window.next_due(&in_flight).is_some());
|
||||
assert_eq!(window.pop_due(&in_flight).expect("released retry").name, "same");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_age_caps_due_time_and_is_not_reset_by_rescheduling() {
|
||||
let mut entry = item("a".to_string(), None);
|
||||
entry.defer(Duration::from_secs(2));
|
||||
let first = entry.first_failure.expect("first failure");
|
||||
assert_eq!(entry.due, first + Duration::from_secs(2));
|
||||
tokio::time::advance(Duration::from_secs(29)).await;
|
||||
entry.defer(Duration::from_secs(8));
|
||||
assert_eq!(entry.first_failure, Some(first));
|
||||
assert_eq!(entry.due, first + MAX_DEFERRED_AGE);
|
||||
assert_eq!(entry.due, first + Duration::from_millis(29_500));
|
||||
assert!(!entry.expired());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert!(entry.expired());
|
||||
|
||||
@@ -157,7 +157,7 @@ async fn wait_for_object_copies(disks: &[PathBuf], bucket: &str, object: &str) {
|
||||
fn deep_heal_opts() -> HealOpts {
|
||||
HealOpts {
|
||||
recreate: true,
|
||||
remove: false,
|
||||
remove: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
}
|
||||
@@ -633,6 +633,7 @@ mod absence_receipt_regressions {
|
||||
version_id: Some(old.to_owned()),
|
||||
},
|
||||
HealOptions {
|
||||
remove_corrupted: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
@@ -823,6 +824,7 @@ mod absence_receipt_regressions {
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
remove_corrupted: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -194,13 +194,21 @@ async fn empty_replacement_with_committed_delete(
|
||||
|
||||
mod serial_tests {
|
||||
use super::*;
|
||||
use storage_api::integration::InstanceContext;
|
||||
|
||||
async fn bucket_directory_loss_task(cluster: bool, missing: usize, dry_run: bool, source: HealRequestSource) {
|
||||
let temp = tempfile::tempdir().expect("isolated bucket recovery fixture");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.instance_context(Arc::new(InstanceContext::new()))
|
||||
.build()
|
||||
.await;
|
||||
let shutdown = env
|
||||
.ecstore
|
||||
.background_cancel_token()
|
||||
.expect("fixture store must own a shutdown token");
|
||||
assert!(!shutdown.is_cancelled(), "a new fixture must not inherit a previous store's shutdown");
|
||||
eprintln!("bucket recovery: environment ready");
|
||||
let bucket = "task-directory-recovery";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
@@ -213,6 +221,7 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("remove only an empty bucket directory");
|
||||
}
|
||||
eprintln!("bucket recovery: directories removed");
|
||||
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).exists()).count(), 4 - missing);
|
||||
let mut request = HealRequest::new(
|
||||
if cluster {
|
||||
@@ -237,6 +246,7 @@ mod serial_tests {
|
||||
}
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
let result = task.execute().await;
|
||||
eprintln!("bucket recovery: task returned");
|
||||
if missing == 4 || source != HealRequestSource::Admin {
|
||||
assert!(
|
||||
result.is_err(),
|
||||
@@ -259,6 +269,7 @@ mod serial_tests {
|
||||
};
|
||||
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).is_dir()).count(), remaining);
|
||||
assert_eq!(storage.admit_bucket_incarnation(bucket).await.expect("retained metadata"), expected);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
@@ -267,7 +278,13 @@ mod serial_tests {
|
||||
for cluster in [false, true] {
|
||||
// Two surviving directories satisfy ordinary reads; one does not.
|
||||
for missing in [2, 3] {
|
||||
bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin).await;
|
||||
eprintln!("bucket recovery: cluster={cluster}, missing={missing}");
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("bucket recovery did not finish: cluster={cluster}, missing={missing}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -810,7 +810,13 @@ fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
|
||||
restarted.start().await.expect("restart with retained failed batch");
|
||||
let restored = restarted.get_task_report(&token).await.expect("original token after restart");
|
||||
assert_eq!(restored.status, report.status);
|
||||
assert_eq!(restored.outcome, report.outcome);
|
||||
// Quorum and Deep observations are local and never persisted.
|
||||
let mut expected_outcome = report.outcome.as_deref().cloned().expect("mixed batch outcome");
|
||||
for object in &mut expected_outcome.objects {
|
||||
object.quorum_selected_data_dir = None;
|
||||
object.deep_check_observed = false;
|
||||
}
|
||||
assert_eq!(restored.outcome.as_deref(), Some(&expected_outcome));
|
||||
assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
|
||||
restarted.stop().await.expect("stop restarted mixed batch manager");
|
||||
});
|
||||
|
||||
@@ -30,6 +30,7 @@ pub(crate) mod integration {
|
||||
ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCommitGuard, WriteCompletion,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rebalance::{RebalStatus, RebalanceMeta, RebalanceStats};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore;
|
||||
pub(crate) use rustfs_storage_api::BucketOperations;
|
||||
pub(crate) use rustfs_storage_api::MakeBucketOptions;
|
||||
|
||||
@@ -382,6 +382,42 @@ impl LockClient for LocalClient {
|
||||
}
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.downgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.try_upgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.refresh(lock_id).await
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let guards = shard.read().await;
|
||||
let Some(entry) = guards.get(lock_id) else { return Ok(None) };
|
||||
if entry.is_expired() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(entry.guard.waiting_writer())
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.release(lock_id).await
|
||||
}
|
||||
@@ -547,6 +583,32 @@ mod tests {
|
||||
assert!(!response.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
|
||||
let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
|
||||
assert!(client.acquire_lock(&first).await.expect("initial lock").success);
|
||||
assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
|
||||
assert!(
|
||||
client
|
||||
.downgrade_committed_tail(&first.lock_id)
|
||||
.await
|
||||
.expect("retry downgrade")
|
||||
);
|
||||
let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
|
||||
.with_acquire_timeout(Duration::from_millis(500));
|
||||
assert!(client.acquire_lock(&reader).await.expect("reader").success);
|
||||
let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
|
||||
assert!(client.release(&first.lock_id).await.expect("release tail"));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
|
||||
assert!(client.release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
|
||||
assert!(client.release(&writer.lock_id).await.expect("release writer"));
|
||||
assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_rejects_a_lock_id_for_another_resource() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
|
||||
@@ -20,6 +20,12 @@ use async_trait::async_trait;
|
||||
use futures::future::join_all;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Marker exchanged in a refresh request and response for a committed-tail conversion.
|
||||
pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
|
||||
pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
|
||||
pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
|
||||
pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
|
||||
|
||||
/// Lock client trait
|
||||
#[async_trait]
|
||||
pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
@@ -48,6 +54,27 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
/// Refresh lock
|
||||
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
/// Convert an exclusive lease to a reader-compatible committed-tail lease.
|
||||
/// Unsupported peers retain their original lease and report `false`.
|
||||
async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership, retaining the writer fence on failure.
|
||||
async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Check support without changing the lease mode.
|
||||
async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Observe foreground writer contention, or return `None` when unsupported.
|
||||
async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Force release lock
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
|
||||
@@ -262,6 +262,8 @@ pub struct DistributedLockGuard {
|
||||
/// All underlying leases that should be refreshed and released with the guard.
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
/// Shared and exclusive quorums must intersect before a tail may convert.
|
||||
committed_tail_quorums_intersect: bool,
|
||||
/// If true, Drop will not try to release (used if user manually released).
|
||||
disarmed: bool,
|
||||
/// Background lease task for renewable leases.
|
||||
@@ -277,7 +279,13 @@ impl DistributedLockGuard {
|
||||
/// - `entries` is the full list of underlying leases that should be refreshed and released
|
||||
/// with this guard.
|
||||
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
|
||||
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
|
||||
fn new(
|
||||
lock_id: LockId,
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
heartbeat: LockHeartbeatConfig,
|
||||
committed_tail_quorums_intersect: bool,
|
||||
) -> Self {
|
||||
record_lock_held_acquire(lock_type);
|
||||
let lock_lost = Arc::new(LockLostSignal::default());
|
||||
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
|
||||
@@ -296,6 +304,7 @@ impl DistributedLockGuard {
|
||||
lock_id,
|
||||
entries,
|
||||
lock_type,
|
||||
committed_tail_quorums_intersect,
|
||||
disarmed: false,
|
||||
refresh_task,
|
||||
lock_lost,
|
||||
@@ -412,6 +421,70 @@ impl DistributedLockGuard {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Convert retained peer leases without ever releasing writer exclusion.
|
||||
/// A peer without conversion support keeps its exclusive lease. Partial
|
||||
/// conversion is safe: both lease modes continue to exclude writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Require every retained lease to confirm exclusivity before committing.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Observe contention on every held lease. Unknown peers prevent a false idle result.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.disarmed || self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
let results =
|
||||
futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
|
||||
if self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
|
||||
return Some(true);
|
||||
}
|
||||
results
|
||||
.iter()
|
||||
.all(|result| matches!(result, Ok(Some(false))))
|
||||
.then_some(false)
|
||||
}
|
||||
|
||||
/// Actual granted peer identities, which may differ from the logical guard ID.
|
||||
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
|
||||
self.entries.iter().map(|entry| &entry.lock_id)
|
||||
@@ -599,6 +672,7 @@ impl DistributedLock {
|
||||
individual_locks,
|
||||
request.lock_type,
|
||||
heartbeat,
|
||||
self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
|
||||
)))
|
||||
} else {
|
||||
// Check if it's a timeout or quorum failure
|
||||
|
||||
@@ -19,8 +19,8 @@ use crate::fast_lock::{
|
||||
use rustfs_io_metrics::{
|
||||
record_read_lock_held_acquire, record_read_lock_held_release, record_write_lock_held_acquire, record_write_lock_held_release,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
/// Global counter for guard IDs to prevent double-release
|
||||
static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1);
|
||||
@@ -39,10 +39,25 @@ pub struct FastLockGuard {
|
||||
shard: Option<Arc<LockShard>>, // None when locks are disabled
|
||||
released: bool,
|
||||
disabled: bool, // True when locks are disabled globally
|
||||
committed_tail: bool,
|
||||
sealed_for_commit: bool,
|
||||
/// Unique ID for this guard instance to prevent double-release
|
||||
guard_id: u64,
|
||||
}
|
||||
|
||||
/// A non-owning check of one local acquisition, including forced release.
|
||||
#[derive(Clone)]
|
||||
pub struct FastLockGuardProbe {
|
||||
shard: Weak<LockShard>,
|
||||
guard_id: u64,
|
||||
}
|
||||
|
||||
impl FastLockGuardProbe {
|
||||
pub fn is_lost(&self) -> bool {
|
||||
self.shard.upgrade().is_none_or(|shard| !shard.is_guard_active(self.guard_id))
|
||||
}
|
||||
}
|
||||
|
||||
impl FastLockGuard {
|
||||
pub(crate) fn new(key: ObjectKey, mode: LockMode, owner: Arc<str>, shard: Arc<LockShard>) -> Self {
|
||||
let guard_id = GUARD_ID_COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -54,6 +69,8 @@ impl FastLockGuard {
|
||||
shard: Some(shard),
|
||||
released: false,
|
||||
disabled: false,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -68,6 +85,8 @@ impl FastLockGuard {
|
||||
shard: None,
|
||||
released: false,
|
||||
disabled: true,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -87,6 +106,83 @@ impl FastLockGuard {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
/// The physical acquisition was force-released before registration.
|
||||
pub(crate) fn abandon_unregistered(&mut self) {
|
||||
if !self.released {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep excluding writers while permitting reads of the committed version.
|
||||
pub fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || self.sealed_for_commit {
|
||||
return false;
|
||||
}
|
||||
if self.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if self.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
if self.disabled {
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
return true;
|
||||
}
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
record_lock_held_release(LockMode::Exclusive);
|
||||
record_lock_held_acquire(LockMode::Shared);
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership without releasing the writer fence.
|
||||
pub fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
|
||||
return false;
|
||||
}
|
||||
if !self.disabled {
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if self.committed_tail {
|
||||
record_lock_held_release(LockMode::Shared);
|
||||
record_lock_held_acquire(LockMode::Exclusive);
|
||||
}
|
||||
self.mode = LockMode::Exclusive;
|
||||
self.committed_tail = false;
|
||||
self.sealed_for_commit = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Observe writers waiting on this guard's exact object key.
|
||||
pub fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.released {
|
||||
return None;
|
||||
}
|
||||
if self.disabled {
|
||||
return Some(false);
|
||||
}
|
||||
self.shard
|
||||
.as_ref()?
|
||||
.waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
|
||||
}
|
||||
|
||||
fn release_physical(&self, shard: &LockShard) -> bool {
|
||||
if self.committed_tail {
|
||||
shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
|
||||
} else {
|
||||
shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
///
|
||||
/// Returns true if the lock was successfully released, false if it was
|
||||
@@ -106,7 +202,7 @@ impl FastLockGuard {
|
||||
}
|
||||
|
||||
if let Some(shard) = &self.shard {
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
let success = self.release_physical(shard);
|
||||
if success {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
@@ -138,9 +234,17 @@ impl FastLockGuard {
|
||||
);
|
||||
}
|
||||
|
||||
/// Check if the lock has been released
|
||||
/// Check if the lock has been released, including by an administrator.
|
||||
pub fn is_released(&self) -> bool {
|
||||
self.released
|
||||
self.released || self.shard.as_ref().is_some_and(|shard| !shard.is_guard_active(self.guard_id))
|
||||
}
|
||||
|
||||
/// Observe this acquisition without extending the lock's lifetime.
|
||||
pub fn release_probe(&self) -> Option<FastLockGuardProbe> {
|
||||
self.shard.as_ref().map(|shard| FastLockGuardProbe {
|
||||
shard: Arc::downgrade(shard),
|
||||
guard_id: self.guard_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if this guard represents a disabled lock
|
||||
@@ -169,7 +273,7 @@ impl Drop for FastLockGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shard) = &self.shard {
|
||||
if !self.released && !self.disabled {
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
let success = self.release_physical(shard);
|
||||
if !success {
|
||||
// For high-concurrency scenarios, this is likely due to:
|
||||
// 1. Lock was already released by another thread
|
||||
|
||||
@@ -96,11 +96,14 @@ impl FastObjectLockManager {
|
||||
/// Acquire object lock
|
||||
pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
|
||||
let shard = self.get_shard(&request.key);
|
||||
match shard.acquire_lock(&request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
match shard.acquire_guarded(&request).await {
|
||||
Ok(acquired) => {
|
||||
let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
// Register guard to prevent premature cleanup
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
|
||||
guard.abandon_unregistered();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
@@ -201,31 +204,15 @@ impl FastObjectLockManager {
|
||||
let mut all_failed = Vec::new();
|
||||
let mut guards = Vec::new();
|
||||
|
||||
for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
for (_, requests) in shard_groups {
|
||||
for request in requests {
|
||||
let key = request.key.clone();
|
||||
let owner = request.owner.clone();
|
||||
let mode = request.mode;
|
||||
|
||||
let acquired = if shard.try_fast_path_only(request) {
|
||||
true
|
||||
} else {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => true,
|
||||
Err(err) => {
|
||||
all_failed.push((key.clone(), err));
|
||||
false
|
||||
}
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
}
|
||||
};
|
||||
|
||||
if acquired {
|
||||
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
Err(err) => all_failed.push((key, err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -245,14 +232,10 @@ impl FastObjectLockManager {
|
||||
let mut acquired_guards = Vec::new();
|
||||
let mut failed_locks = Vec::new();
|
||||
|
||||
'outer: for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
'outer: for (_, requests) in shard_groups {
|
||||
for request in requests {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
acquired_guards.push(guard);
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
@@ -94,6 +94,7 @@ impl ObjectLockState {
|
||||
pub fn reset_for_reuse(&mut self) {
|
||||
// Reset atomic state
|
||||
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
|
||||
self.force_epoch.store(0, Ordering::Relaxed);
|
||||
|
||||
// Clear owners
|
||||
*self.current_owner.write() = None;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -30,6 +31,12 @@ struct ActiveGuardInfo {
|
||||
key: ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: Arc<str>,
|
||||
committed_tail: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct AcquiredLock {
|
||||
state: Arc<ObjectLockState>,
|
||||
force_epoch: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
@@ -46,6 +53,8 @@ enum TestPausePoint {
|
||||
BeforeFastWrite,
|
||||
FastWriteExisting,
|
||||
SlowLookup,
|
||||
ForceAfterTailRelease,
|
||||
ReleaseAfterUnregister,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -69,7 +78,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
|
||||
|
||||
/// Lock shard to reduce global contention.
|
||||
///
|
||||
/// When both are needed, lock `objects` before a state's owner metadata.
|
||||
/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
|
||||
#[derive(Debug)]
|
||||
pub struct LockShard {
|
||||
/// Object lock states - using parking_lot for better performance
|
||||
@@ -133,12 +142,16 @@ impl LockShard {
|
||||
|
||||
/// Acquire lock with fast path optimization
|
||||
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
|
||||
self.acquire_guarded(request).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
|
||||
let start_time = Instant::now();
|
||||
|
||||
// Try fast path first
|
||||
if let Some(_state) = self.try_fast_path(request) {
|
||||
if let Some(acquired) = self.try_fast_path(request) {
|
||||
self.metrics.record_fast_path_success();
|
||||
return Ok(());
|
||||
return Ok(acquired);
|
||||
}
|
||||
|
||||
// Slow path with waiting
|
||||
@@ -157,7 +170,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Try fast path lock acquisition (lock-free when possible)
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
|
||||
// First try to get existing state without write lock
|
||||
{
|
||||
let objects = self.objects.read();
|
||||
@@ -168,14 +181,16 @@ impl LockShard {
|
||||
test_pause(TestPausePoint::FastExisting, &request.key);
|
||||
|
||||
// Try atomic acquisition
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
|
||||
if success {
|
||||
return Some(state);
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,10 +210,11 @@ impl LockShard {
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::FastWriteExisting, &request.key);
|
||||
|
||||
let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
drop(objects);
|
||||
if acquired {
|
||||
return Some(state);
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
}
|
||||
} else {
|
||||
// Create new state from pool and acquire immediately
|
||||
@@ -206,7 +222,7 @@ impl LockShard {
|
||||
let state = Arc::new(*state_box);
|
||||
if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
|
||||
objects.insert(request.key.clone(), state.clone());
|
||||
return Some(state);
|
||||
return Some(AcquiredLock { state, force_epoch: 0 });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -215,7 +231,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Slow path with async waiting
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
|
||||
// Use adaptive timeout based on current load and request priority
|
||||
let adaptive_timeout = self.calculate_adaptive_timeout(request);
|
||||
let deadline = start_time + adaptive_timeout;
|
||||
@@ -234,7 +250,7 @@ impl LockShard {
|
||||
|
||||
loop {
|
||||
// Keep the state in the map until its acquisition attempt completes.
|
||||
let (state, success) = {
|
||||
let (state, success, force_epoch) = {
|
||||
let mut objects = self.objects.write();
|
||||
let state = match objects.get(&request.key) {
|
||||
Some(state) => state.clone(),
|
||||
@@ -249,17 +265,19 @@ impl LockShard {
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::SlowLookup, &request.key);
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
(state, success)
|
||||
(state, success, force_epoch)
|
||||
};
|
||||
|
||||
if success {
|
||||
self.metrics.record_slow_path_success();
|
||||
return Ok(());
|
||||
return Ok(AcquiredLock { state, force_epoch });
|
||||
}
|
||||
|
||||
// Check timeout
|
||||
@@ -381,14 +399,10 @@ impl LockShard {
|
||||
|
||||
/// Release lock with guard ID tracking for double-release prevention
|
||||
pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
|
||||
// First, try to remove the guard from active set
|
||||
let guard_was_active = {
|
||||
let mut guards = self.active_guards.lock();
|
||||
guards.remove(&guard_id).is_some()
|
||||
};
|
||||
|
||||
// If guard was not active, this is a double-release attempt
|
||||
if !guard_was_active {
|
||||
// Keep force-unlock from releasing this holder and admitting a new
|
||||
// same-owner holder between guard removal and physical release.
|
||||
let mut guards = self.active_guards.lock();
|
||||
if guards.remove(&guard_id).is_none() {
|
||||
tracing::debug!(
|
||||
"Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
|
||||
key,
|
||||
@@ -399,6 +413,9 @@ impl LockShard {
|
||||
return false;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ReleaseAfterUnregister, key);
|
||||
|
||||
// Proceed with normal release
|
||||
let should_cleanup;
|
||||
let result;
|
||||
@@ -423,6 +440,8 @@ impl LockShard {
|
||||
}
|
||||
}
|
||||
|
||||
drop(guards);
|
||||
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
@@ -430,21 +449,110 @@ impl LockShard {
|
||||
result
|
||||
}
|
||||
|
||||
/// Convert an active exclusive guard without opening a writer gap.
|
||||
pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.downgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Shared;
|
||||
info.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return false;
|
||||
}
|
||||
if info.mode == LockMode::Exclusive && !info.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.try_upgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Exclusive;
|
||||
info.committed_tail = false;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
|
||||
let guards = self.active_guards.lock();
|
||||
let info = guards.get(&guard_id)?.as_ref()?;
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return None;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let state = objects.get(key)?;
|
||||
Some(state.atomic_state.writer_observed_during_tail())
|
||||
}
|
||||
|
||||
/// Release a converted guard and its special reader-admission state.
|
||||
pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let (success, should_cleanup) = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
let success = state.release_committed_tail(owner);
|
||||
(success, success && !state.is_locked() && !state.atomic_state.has_waiters())
|
||||
};
|
||||
if success {
|
||||
guards.remove(&guard_id);
|
||||
self.metrics.record_release();
|
||||
}
|
||||
drop(guards);
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
success
|
||||
}
|
||||
|
||||
/// Register a guard to prevent premature cleanup
|
||||
pub fn register_guard(&self, guard_id: u64) {
|
||||
self.active_guards.lock().insert(guard_id, None);
|
||||
}
|
||||
|
||||
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
|
||||
pub(crate) fn register_guard_with_info(
|
||||
&self,
|
||||
guard_id: u64,
|
||||
key: &ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: &Arc<str>,
|
||||
acquired: &AcquiredLock,
|
||||
) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.read();
|
||||
if !objects.get(key).is_some_and(|state| {
|
||||
Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
guards.insert(
|
||||
guard_id,
|
||||
Some(ActiveGuardInfo {
|
||||
key: key.clone(),
|
||||
mode,
|
||||
owner: owner.clone(),
|
||||
committed_tail: false,
|
||||
}),
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Unregister a guard (called when guard is dropped)
|
||||
@@ -460,9 +568,8 @@ impl LockShard {
|
||||
guards.len()
|
||||
}
|
||||
|
||||
/// Check if a guard is active (for testing)
|
||||
#[cfg(test)]
|
||||
pub fn is_guard_active(&self, guard_id: u64) -> bool {
|
||||
/// Check whether this exact guard still owns its acquisition.
|
||||
pub(crate) fn is_guard_active(&self, guard_id: u64) -> bool {
|
||||
let guards = self.active_guards.lock();
|
||||
guards.contains_key(&guard_id)
|
||||
}
|
||||
@@ -719,27 +826,57 @@ impl LockShard {
|
||||
/// Returns the number of owners that were released. Used by the admin
|
||||
/// force-unlock path to clear a stuck resource.
|
||||
pub fn force_release_all(&self, key: &ObjectKey) -> usize {
|
||||
let owners_modes: Vec<(Arc<str>, LockMode)> = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else {
|
||||
return 0;
|
||||
};
|
||||
let mut pairs = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
pairs.push((info.owner, LockMode::Exclusive));
|
||||
}
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
pairs.push((entry.owner.clone(), LockMode::Shared));
|
||||
}
|
||||
pairs
|
||||
};
|
||||
|
||||
// Serialize tail conversion and prevent new acquisitions while taking
|
||||
// and releasing the owner snapshot. A new tail must never be mistaken
|
||||
// for an ordinary shared owner by force unlock.
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.write();
|
||||
let Some(state) = objects.get(key) else { return 0 };
|
||||
// Acquisitions completed before this point but not yet registered
|
||||
// must not later create guards for the released generation.
|
||||
let _ = state
|
||||
.force_epoch
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
|
||||
let tail = guards.iter().find_map(|(&id, info)| {
|
||||
info.as_ref()
|
||||
.filter(|info| info.committed_tail && info.key == *key)
|
||||
.map(|info| (id, info.owner.clone()))
|
||||
});
|
||||
let mut released = 0;
|
||||
for (owner, mode) in owners_modes {
|
||||
if self.release_lock(key, &owner, mode) {
|
||||
if let Some((id, owner)) = tail
|
||||
&& state.release_committed_tail(&owner)
|
||||
{
|
||||
guards.remove(&id);
|
||||
self.metrics.record_release();
|
||||
released += 1;
|
||||
}
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ForceAfterTailRelease, key);
|
||||
let mut owners_modes = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
owners_modes.push((info.owner, LockMode::Exclusive, 1));
|
||||
}
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
|
||||
}
|
||||
for (owner, mode, count) in owners_modes {
|
||||
let mut owner_released = false;
|
||||
for _ in 0..count {
|
||||
let success = match mode {
|
||||
LockMode::Shared => state.release_shared(&owner),
|
||||
LockMode::Exclusive => state.release_exclusive(&owner),
|
||||
};
|
||||
if success {
|
||||
self.metrics.record_release();
|
||||
owner_released = true;
|
||||
}
|
||||
}
|
||||
if owner_released {
|
||||
released += 1;
|
||||
}
|
||||
}
|
||||
// Old guard drops must not release a later holder with the same owner.
|
||||
guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
|
||||
released
|
||||
}
|
||||
|
||||
@@ -979,6 +1116,320 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
|
||||
for reader in [false, true] {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-release-gap");
|
||||
let old = if reader {
|
||||
manager.acquire_read_lock(key.clone(), "same-owner").await
|
||||
} else {
|
||||
manager.acquire_write_lock(key.clone(), "same-owner").await
|
||||
}
|
||||
.expect("old holder");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
|
||||
let old_drop = tokio::task::spawn_blocking(move || drop(old));
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
|
||||
|
||||
let can_force = shard.active_guards.try_lock().is_some();
|
||||
let mut force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
let mut new_guard = if can_force {
|
||||
assert_eq!(force.await.expect("force join"), 1);
|
||||
let mut guard = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
if reader {
|
||||
assert!(guard.downgrade_committed_tail());
|
||||
}
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
guard
|
||||
} else {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
assert_eq!(force.await.expect("force join"), 0);
|
||||
manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer")
|
||||
};
|
||||
if reader && !can_force {
|
||||
assert!(new_guard.downgrade_committed_tail());
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old release must not unlock the new writer"
|
||||
);
|
||||
drop(new_guard);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_does_not_capture_a_new_committed_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-tail-race");
|
||||
let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
|
||||
let force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
|
||||
let mut next = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
|
||||
resume.send(()).expect("resume force unlock");
|
||||
assert_eq!(force.await.expect("force unlock task"), 1);
|
||||
let mut new = next.await.expect("new writer task").expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"the new committed tail must still exclude writers"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-tail");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager
|
||||
.acquire_read_lock(key.clone(), "reader")
|
||||
.await
|
||||
.expect("reader during tail");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
|
||||
.await
|
||||
.is_err(),
|
||||
"new readers must not starve a pending upgrade"
|
||||
);
|
||||
drop(reader);
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail(),
|
||||
"a delayed downgrade must not reopen reads after upgrade"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
drop(tail);
|
||||
let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "brief-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "waiting-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(tail.waiting_writer(), Some(false));
|
||||
let writer = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer() != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("writer becomes observable");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
|
||||
let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_a_pending_upgrade_allows_readers_again() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "cancel-upgrade");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
let mut waiting = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
|
||||
drop(tail);
|
||||
let second = tokio::time::timeout(Duration::from_secs(2), waiting)
|
||||
.await
|
||||
.expect("reader wakes after cancellation")
|
||||
.expect("reader task joins")
|
||||
.expect("reader obtains lock");
|
||||
drop((reader, second));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-stale");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "late-downgrade");
|
||||
let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
assert!(!guard.downgrade_committed_tail());
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_readers_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-old-readers");
|
||||
let old1 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("first reader");
|
||||
let old2 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("second reader");
|
||||
assert!(!old1.is_released());
|
||||
assert!(!old2.is_released());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(old1.is_released());
|
||||
assert!(old2.is_released());
|
||||
assert!(!tail.is_released());
|
||||
drop((old1, old2));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"old reader drops must not release the new tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_unregistered_reader_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-unregistered-reader");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard
|
||||
.acquire_guarded(&request)
|
||||
.await
|
||||
.expect("old reader acquired before registration");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
|
||||
old.abandon_unregistered();
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old reader must not release a new committed tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "unrelated-acquisition");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
|
||||
let other = (0..1000)
|
||||
.map(|index| ObjectKey::new("bucket", format!("other-{index}")))
|
||||
.find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
|
||||
.expect("another key on the same shard");
|
||||
let other_guard = manager
|
||||
.acquire_write_lock(other.clone(), "other")
|
||||
.await
|
||||
.expect("other writer");
|
||||
assert_eq!(manager.force_unlock(&other), 1);
|
||||
drop(other_guard);
|
||||
let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
|
||||
drop(guard);
|
||||
assert!(manager.acquire_write_lock(key, "next").await.is_ok());
|
||||
}
|
||||
|
||||
fn start_acquisition(
|
||||
manager: &Arc<FastObjectLockManager>,
|
||||
request: ObjectLockRequest,
|
||||
|
||||
@@ -23,6 +23,11 @@ use crate::fast_lock::types::{LockMode, LockPriority};
|
||||
/// Optimized atomic lock state encoding in u64
|
||||
/// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
|
||||
const WRITER_FLAG_MASK: u64 = 0x1;
|
||||
// A committed write tail is a reader-compatible owner that still excludes writers.
|
||||
const COMMITTED_TAIL_FLAG: u64 = 0x2;
|
||||
const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
|
||||
const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
|
||||
const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
|
||||
const READERS_SHIFT: u8 = 8;
|
||||
const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
|
||||
const READERS_WAITING_SHIFT: u8 = 16;
|
||||
@@ -31,7 +36,6 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
|
||||
const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
|
||||
|
||||
// Fast path check masks
|
||||
const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
|
||||
const COMPLETELY_UNLOCKED: u64 = 0;
|
||||
|
||||
/// Fast atomic lock state for single version
|
||||
@@ -66,8 +70,8 @@ impl AtomicLockState {
|
||||
let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
|
||||
match mode {
|
||||
LockMode::Shared => {
|
||||
// No writer and no waiting writers
|
||||
(state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
|
||||
(state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
|
||||
&& ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
|
||||
}
|
||||
LockMode::Exclusive => {
|
||||
// Completely unlocked
|
||||
@@ -83,8 +87,11 @@ impl AtomicLockState {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
|
||||
// Fast path check - cannot acquire if there's a writer or writers waiting
|
||||
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
|
||||
// While a committed tail owns a shared slot, let reads through
|
||||
// even if a subsequent writer is waiting for that tail to drain.
|
||||
if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
|
||||
|| ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -137,6 +144,78 @@ impl AtomicLockState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically retain writer exclusion while allowing reads of a committed tail.
|
||||
fn downgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
|
||||
return false;
|
||||
}
|
||||
let next = (current & !WRITER_FLAG_MASK)
|
||||
| COMMITTED_TAIL_FLAG
|
||||
| (1 << READERS_SHIFT)
|
||||
| if current & WRITERS_WAITING_MASK != 0 {
|
||||
TAIL_WRITER_OBSERVED_FLAG
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore exclusive ownership only after every other reader has left.
|
||||
fn try_upgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
|
||||
return false;
|
||||
}
|
||||
let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
|
||||
let next = if readers_drained {
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
|
||||
| WRITER_FLAG_MASK
|
||||
| TAIL_TRACK_WRITERS_FLAG
|
||||
} else {
|
||||
current | TAIL_UPGRADE_PENDING_FLAG
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return readers_drained;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release the committed tail's own shared slot and restore ordinary fairness.
|
||||
fn release_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
|
||||
return false;
|
||||
}
|
||||
let next =
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self) -> bool {
|
||||
loop {
|
||||
@@ -169,7 +248,7 @@ impl AtomicLockState {
|
||||
return false; // No exclusive lock to release
|
||||
}
|
||||
|
||||
let new_state = current & !WRITER_FLAG_MASK;
|
||||
let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -236,7 +315,10 @@ impl AtomicLockState {
|
||||
return false; // Max waiting writers
|
||||
}
|
||||
|
||||
let new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
|
||||
new_state |= TAIL_WRITER_OBSERVED_FLAG;
|
||||
}
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -321,10 +403,15 @@ impl AtomicLockState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn writers_waiting_count(&self) -> u16 {
|
||||
pub(crate) fn writers_waiting_count(&self) -> u16 {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state)
|
||||
}
|
||||
|
||||
pub(crate) fn writer_observed_during_tail(&self) -> bool {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Object lock state with version support - optimized memory layout
|
||||
@@ -334,6 +421,8 @@ pub struct ObjectLockState {
|
||||
// First cache line: Most frequently accessed data
|
||||
/// Atomic state for fast operations
|
||||
pub atomic_state: AtomicLockState,
|
||||
/// Changes whenever an administrator force-releases this state.
|
||||
pub(crate) force_epoch: AtomicU64,
|
||||
|
||||
// Second cache line: Notification mechanisms
|
||||
/// Notification for readers (traditional)
|
||||
@@ -377,6 +466,7 @@ impl ObjectLockState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
atomic_state: AtomicLockState::new(),
|
||||
force_epoch: AtomicU64::new(0),
|
||||
read_notify: Notify::new(),
|
||||
write_notify: Notify::new(),
|
||||
optimized_notify: OptimizedNotify::new(),
|
||||
@@ -425,6 +515,70 @@ impl ObjectLockState {
|
||||
true
|
||||
}
|
||||
|
||||
/// Convert the current exclusive owner into the committed-tail owner.
|
||||
pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
|
||||
return false;
|
||||
};
|
||||
let mut shared = self.shared_owners.write();
|
||||
if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
shared.push(SharedOwnerEntry {
|
||||
owner: info.owner,
|
||||
count: 1,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
*current = None;
|
||||
drop(shared);
|
||||
drop(current);
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
let info = shared.remove(pos);
|
||||
*current = Some(ExclusiveOwnerInfo {
|
||||
owner: info.owner,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Release the committed-tail owner's shared slot without releasing readers.
|
||||
pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if !self.atomic_state.release_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
if shared[pos].count > 1 {
|
||||
shared[pos].count -= 1;
|
||||
} else {
|
||||
shared.remove(pos);
|
||||
}
|
||||
let no_readers = shared.is_empty();
|
||||
drop(shared);
|
||||
if no_readers {
|
||||
self.optimized_notify.notify_writer();
|
||||
}
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
@@ -555,6 +709,24 @@ mod tests {
|
||||
assert!(!state.release_exclusive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
|
||||
let state = AtomicLockState::new();
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.inc_writers_waiting());
|
||||
assert!(state.downgrade_committed_tail());
|
||||
assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
|
||||
assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
|
||||
assert!(state.release_committed_tail());
|
||||
assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
|
||||
assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
|
||||
assert!(state.release_shared());
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.release_exclusive());
|
||||
state.dec_writers_waiting();
|
||||
assert!(state.is_free());
|
||||
}
|
||||
|
||||
// Regression for the waiter-preserving exclusive CAS.
|
||||
//
|
||||
// The acquisition CAS used to demand a fully-zero state word, which
|
||||
|
||||
@@ -111,6 +111,67 @@ mod fast_lock_tests {
|
||||
assert!(guard.is_released(), "Guard should be marked as released");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_tail_cannot_release_new_tail() {
|
||||
let manager = create_test_manager();
|
||||
let key = ObjectKey::new("test-bucket", "force-tail-generation");
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let mut old = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut new = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
let attempt =
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
|
||||
.await;
|
||||
assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
|
||||
drop(new);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_allows_reader_while_writer_waits() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
let key = ObjectKey::new("test-bucket", "tail-handoff");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), Arc::from("first-writer"))
|
||||
.await
|
||||
.expect("initial writer");
|
||||
let writer = tokio::spawn({
|
||||
let manager = Arc::clone(&manager);
|
||||
let key = key.clone();
|
||||
async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while manager.current_lock_counts().1 == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second writer should be queued");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
|
||||
.await
|
||||
.expect("reader should not wait behind committed tail")
|
||||
.expect("reader should acquire");
|
||||
assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
|
||||
drop(tail);
|
||||
assert!(!writer.is_finished(), "reader still excludes the next writer");
|
||||
drop(reader);
|
||||
let writer = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer should resume")
|
||||
.expect("writer task should join")
|
||||
.expect("writer should acquire");
|
||||
drop(writer);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_lock_counts_report_live_holders_and_waiters() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
|
||||
@@ -107,6 +107,38 @@ impl NamespaceLockGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Permit reads while continuing to exclude competing writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.downgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.downgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to restore exclusive access while retaining the writer fence.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.try_upgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the held leases can safely use heal lock transitions.
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.supports_heal_lock_transition().await,
|
||||
Self::Fast(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe whether a writer is waiting on this object's held leases.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.waiting_writer().await,
|
||||
Self::Fast(guard) => guard.waiting_writer(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
pub fn release(&mut self) -> bool {
|
||||
match self {
|
||||
|
||||
@@ -154,6 +154,43 @@ impl crate::client::LockClient for DelayedClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingTailClient(LocalClient);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::client::LockClient for PendingTailClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
self.0.acquire_lock(request).await
|
||||
}
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.release(lock_id).await
|
||||
}
|
||||
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.refresh(lock_id).await
|
||||
}
|
||||
async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
|
||||
std::future::pending().await
|
||||
}
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.force_release(lock_id).await
|
||||
}
|
||||
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
self.0.check_status(lock_id).await
|
||||
}
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
self.0.get_stats().await
|
||||
}
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
self.0.close().await
|
||||
}
|
||||
async fn is_online(&self) -> bool {
|
||||
self.0.is_online().await
|
||||
}
|
||||
async fn is_local(&self) -> bool {
|
||||
self.0.is_local().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FlakyAcquireClient {
|
||||
inner: LocalClient,
|
||||
@@ -579,6 +616,286 @@ async fn test_namespace_lock_guard_lock_id() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_conversion_rejects_non_intersecting_quorums() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
|
||||
let resource = ObjectKey::new("bucket", "non-intersecting-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lease");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail().await,
|
||||
"conversion requires intersecting read and write quorums"
|
||||
);
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
|
||||
for index in 0..4 {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
if index < 2 {
|
||||
clients.push(inner);
|
||||
} else {
|
||||
clients.push(Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "partial-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"mixed leases still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
|
||||
let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
|
||||
let resource = ObjectKey::new("bucket", "cancel-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("other peers converted before cancellation");
|
||||
assert!(reader.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"cancelled conversion must still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_peer_keeps_exclusive_tail_lease() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}) as Arc<dyn crate::client::LockClient>
|
||||
})
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "old-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
|
||||
assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
|
||||
assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after tail");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "committed-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("committed reader");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"tail and reader must exclude writer"
|
||||
);
|
||||
assert!(tail.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"reader still excludes writer"
|
||||
);
|
||||
assert!(reader.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after drain");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "tail-upgrade");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.supports_heal_lock_transition().await);
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("reader");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
|
||||
assert!(reader.release());
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !tail.try_upgrade_committed_tail().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry must finish partial upgrades after asynchronous reader release");
|
||||
assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_observes_a_waiting_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
|
||||
let resource = ObjectKey::new("bucket", "writer-observation");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
assert_eq!(tail.waiting_writer().await, Some(false));
|
||||
let writer = {
|
||||
let lock = Arc::clone(&lock);
|
||||
let resource = resource.clone();
|
||||
tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer().await != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiting writer becomes visible on a held peer");
|
||||
assert!(tail.release());
|
||||
let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
assert!(guard.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
|
||||
use crate::client::LockClient;
|
||||
|
||||
let clients: Vec<Arc<LocalClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients(
|
||||
"partial-upgrade-writer".to_owned(),
|
||||
clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
|
||||
);
|
||||
let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
let ids = match &tail {
|
||||
NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
|
||||
NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
|
||||
};
|
||||
let mut holders = Vec::new();
|
||||
for client in &clients {
|
||||
for id in &ids {
|
||||
if client.check_status(id).await.expect("peer status").is_some() {
|
||||
holders.push(Arc::clone(client));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(holders.len() >= 2, "write quorum must hold multiple peers");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
|
||||
assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
|
||||
let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
|
||||
assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
|
||||
assert_eq!(
|
||||
tail.waiting_writer().await,
|
||||
Some(true),
|
||||
"partial upgrade must retain the earlier contention"
|
||||
);
|
||||
assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_multi_node_simulation() {
|
||||
// Simulate a 3-node distributed environment where each node has its own lock backend
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1092,6 +1092,56 @@ impl_canonical_mutation_body!(
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
// the JSON fallback decode path, breaks the signature-bound digest.
|
||||
|
||||
pub fn canonical_metadata_reservation_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_u64(request.timeout_ms);
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_cancel_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationCancelRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_activate_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationActivateRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
// Keep ordinary activations byte-compatible with existing peers. Renewal
|
||||
// adds a distinct signed marker so an older peer cannot treat it as activation.
|
||||
if request.renew_only {
|
||||
body.push_bool(true);
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_reserved_rename_data_request_body(
|
||||
request: &proto_gen::node_service::ReservedRenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
|
||||
if let Some(rename) = request.rename.as_ref() {
|
||||
body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
|
||||
} else {
|
||||
body.push_bytes(&[])?;
|
||||
}
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -1722,6 +1772,96 @@ mod disk_mutation_canonical_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
|
||||
use proto_gen::node_service::{
|
||||
MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
ReservedRenameDataRequest,
|
||||
};
|
||||
|
||||
let request = MetadataReservationRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.timeout_ms = 1,
|
||||
|r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let activation = MetadataReservationActivateRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
token: vec![1; 16].into(),
|
||||
renew_only: false,
|
||||
};
|
||||
let mut legacy_activation = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
|
||||
legacy_activation.push_str(&activation.disk).expect("disk");
|
||||
legacy_activation.push_str(&activation.bucket).expect("bucket");
|
||||
legacy_activation.push_str(&activation.object).expect("object");
|
||||
legacy_activation.push_bytes(&activation.token).expect("token");
|
||||
assert_eq!(
|
||||
canonical_metadata_reservation_activate_request_body(&activation).expect("ordinary activation"),
|
||||
legacy_activation.finish(),
|
||||
"ordinary activation must retain its previous body digest"
|
||||
);
|
||||
let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.renew_only = true,
|
||||
] {
|
||||
let mut changed = activation.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let cancellation = MetadataReservationCancelRequest {
|
||||
disk: "disk".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed = cancellation.clone();
|
||||
changed.token = vec![2; 16].into();
|
||||
assert_ne!(
|
||||
canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
|
||||
canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
|
||||
);
|
||||
let rename = ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
dst_path: "object".into(),
|
||||
..Default::default()
|
||||
}),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed_token = rename.clone();
|
||||
changed_token.token = vec![2; 16].into();
|
||||
let mut changed_object = rename.clone();
|
||||
changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
|
||||
assert_all_distinct(&[
|
||||
canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
|
||||
canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
|
||||
canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
|
||||
canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
|
||||
rename: None,
|
||||
token: rename.token,
|
||||
})
|
||||
.expect("missing rename"),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
|
||||
@@ -339,6 +339,32 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
message MetadataReservationRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
uint64 timeout_ms = 4;
|
||||
bytes bucket_incarnation_id = 5;
|
||||
}
|
||||
|
||||
message MetadataReservationCancelRequest {
|
||||
string disk = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
message MetadataReservationActivateRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
bytes token = 4;
|
||||
bool renew_only = 5;
|
||||
}
|
||||
|
||||
message ReservedRenameDataRequest {
|
||||
RenameDataRequest rename = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1314,6 +1340,11 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ReserveObjectMetadataShort(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -1081,7 +1081,7 @@ impl ChecksumHasher for Sha512Hasher {
|
||||
/// MD5 hasher for the ADDITIONAL checksum (x-amz-checksum-md5). Separate from the
|
||||
/// legacy Content-MD5 / ETag machinery — this only serves the flexible-checksum path.
|
||||
pub struct Md5Hasher {
|
||||
hasher: md5::Md5,
|
||||
hasher: rustfs_utils::FastMd5,
|
||||
}
|
||||
|
||||
impl Default for Md5Hasher {
|
||||
@@ -1092,14 +1092,14 @@ impl Default for Md5Hasher {
|
||||
|
||||
impl Md5Hasher {
|
||||
pub fn new() -> Self {
|
||||
use md5::Digest as _;
|
||||
Self { hasher: md5::Md5::new() }
|
||||
Self {
|
||||
hasher: rustfs_utils::FastMd5::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Md5Hasher {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
use md5::Digest as _;
|
||||
self.hasher.update(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
@@ -1111,13 +1111,11 @@ impl Write for Md5Hasher {
|
||||
|
||||
impl ChecksumHasher for Md5Hasher {
|
||||
fn finalize(&mut self) -> Vec<u8> {
|
||||
use md5::Digest as _;
|
||||
self.hasher.clone().finalize().to_vec()
|
||||
self.hasher.finalize_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
use md5::Digest as _;
|
||||
self.hasher = md5::Md5::new();
|
||||
self.hasher.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+102
-28
@@ -15,8 +15,8 @@
|
||||
use crate::compress_index::{Index, TryGetIndex};
|
||||
use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use md5::{Digest, Md5};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::FastMd5;
|
||||
use sha2::Sha256;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
@@ -27,11 +27,12 @@ pin_project! {
|
||||
pub struct EtagReader<R> {
|
||||
#[pin]
|
||||
pub inner: R,
|
||||
pub md5: Md5,
|
||||
pub md5: FastMd5,
|
||||
pub finished: bool,
|
||||
pub checksum: Option<String>,
|
||||
resolved_etag: Option<String>,
|
||||
encrypted_etag_mac: Option<Hmac<Sha256>>,
|
||||
delegate_inner_etag: bool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,11 +40,12 @@ impl<R> EtagReader<R> {
|
||||
pub fn new(inner: R, checksum: Option<String>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
md5: Md5::new(),
|
||||
md5: FastMd5::new(),
|
||||
finished: false,
|
||||
checksum,
|
||||
resolved_etag: None,
|
||||
encrypted_etag_mac: None,
|
||||
delegate_inner_etag: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +60,17 @@ impl<R> EtagReader<R> {
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
/// Reuse the inner reader's plaintext ETag when deriving an encrypted ETag,
|
||||
/// avoiding a redundant second MD5 pass over the same plaintext stream.
|
||||
pub fn with_inner_etag_delegation(mut self, delegate: bool) -> Self
|
||||
where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
self.delegate_inner_etag =
|
||||
delegate && self.encrypted_etag_mac.is_some() && self.checksum.is_none() && self.inner.is_etag_reader();
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the final md5 value (etag) as a hex string, only compute once.
|
||||
/// Can be called multiple times, always returns the same result after finished.
|
||||
pub fn get_etag(&mut self) -> String {
|
||||
@@ -65,8 +78,8 @@ impl<R> EtagReader<R> {
|
||||
return etag.clone();
|
||||
}
|
||||
|
||||
let etag = self.md5.clone().finalize().to_vec();
|
||||
let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let digest = self.md5.finalize_bytes();
|
||||
let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
self.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
}
|
||||
@@ -87,30 +100,32 @@ where
|
||||
if let Poll::Ready(Ok(())) = &poll {
|
||||
let filled = &buf.filled()[orig_filled..];
|
||||
if !filled.is_empty() {
|
||||
this.md5.update(filled);
|
||||
if !*this.delegate_inner_etag {
|
||||
this.md5.update(filled);
|
||||
}
|
||||
} else {
|
||||
// EOF
|
||||
*this.finished = true;
|
||||
let etag = if let Some(etag) = this.resolved_etag.as_ref() {
|
||||
etag.clone()
|
||||
} else {
|
||||
let etag = this.md5.clone().finalize().to_vec();
|
||||
let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
*this.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
};
|
||||
if let Some(checksum) = this.checksum {
|
||||
let etag = if let Some(etag) = this.resolved_etag.as_ref() {
|
||||
etag.clone()
|
||||
} else {
|
||||
let digest = this.md5.finalize_bytes();
|
||||
let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
*this.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
};
|
||||
|
||||
if let Some(checksum) = this.checksum
|
||||
&& *checksum != etag
|
||||
{
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
BadDigest {
|
||||
expected_md5: checksum.clone(),
|
||||
calculated_md5: etag,
|
||||
},
|
||||
)));
|
||||
if *checksum != etag {
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
BadDigest {
|
||||
expected_md5: checksum.clone(),
|
||||
calculated_md5: etag,
|
||||
},
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,16 +133,24 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> EtagResolvable for EtagReader<R> {
|
||||
impl<R> EtagResolvable for EtagReader<R>
|
||||
where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
true
|
||||
!self.delegate_inner_etag || self.inner.is_etag_reader()
|
||||
}
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
if let Some(mut mac) = self.encrypted_etag_mac.clone() {
|
||||
if !self.finished {
|
||||
return None;
|
||||
}
|
||||
mac.update(self.get_etag().as_bytes());
|
||||
let plaintext_etag = if self.delegate_inner_etag {
|
||||
self.inner.try_resolve_etag()?
|
||||
} else {
|
||||
self.get_etag()
|
||||
};
|
||||
mac.update(plaintext_etag.as_bytes());
|
||||
let protected = mac.finalize().into_bytes();
|
||||
return Some(hex_simd::encode_to_string(&protected[..16], hex_simd::AsciiCase::Lower));
|
||||
}
|
||||
@@ -167,6 +190,7 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use md5::{Digest, Md5};
|
||||
use rand::RngExt;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
@@ -340,4 +364,54 @@ mod tests {
|
||||
assert_eq!(digest.expected_md5, wrong_checksum);
|
||||
assert_eq!(digest.calculated_md5, calculated_md5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_etag_reader_with_inner_delegation_matches_standalone() {
|
||||
let data = b"delegated encrypted etag payload across multiple 64-byte MD5 blocks for verification";
|
||||
let key = [0x42; 32];
|
||||
let nonce = [0x24; 12];
|
||||
|
||||
let mut standalone =
|
||||
EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce).expect("standalone encrypted EtagReader");
|
||||
let mut out_standalone = Vec::new();
|
||||
standalone.read_to_end(&mut out_standalone).await.expect("read standalone");
|
||||
let expected_tag = standalone.try_resolve_etag().expect("standalone etag");
|
||||
|
||||
let inner =
|
||||
crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
|
||||
.expect("inner HashReader");
|
||||
assert!(inner.is_etag_reader(), "HashReader with diskable_md5=false must report is_etag_reader");
|
||||
let mut delegated = EtagReader::new_encrypted(inner, key, nonce)
|
||||
.expect("delegated encrypted EtagReader")
|
||||
.with_inner_etag_delegation(true);
|
||||
let mut out_delegated = Vec::new();
|
||||
delegated.read_to_end(&mut out_delegated).await.expect("read delegated");
|
||||
assert_eq!(out_delegated, data);
|
||||
assert_eq!(
|
||||
delegated.try_resolve_etag().as_deref(),
|
||||
Some(expected_tag.as_str()),
|
||||
"delegated encrypted EtagReader must produce identical HMAC ETag without re-hashing plaintext"
|
||||
);
|
||||
|
||||
let truncated = crate::LimitReader::new(
|
||||
crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
|
||||
.expect("inner HashReader for limit"),
|
||||
16,
|
||||
);
|
||||
assert!(
|
||||
!truncated.is_etag_reader(),
|
||||
"truncating LimitReader must not advertise inner ETag delegation"
|
||||
);
|
||||
|
||||
let mut non_etag_inner = EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce)
|
||||
.expect("encrypted EtagReader over Cursor")
|
||||
.with_inner_etag_delegation(true);
|
||||
let mut out_cursor = Vec::new();
|
||||
non_etag_inner.read_to_end(&mut out_cursor).await.expect("read cursor");
|
||||
assert_eq!(
|
||||
non_etag_inner.try_resolve_etag().as_deref(),
|
||||
Some(expected_tag.as_str()),
|
||||
"delegation request on non-EtagResolvable inner must keep outer MD5 hashing active"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -630,6 +630,10 @@ impl AsyncRead for HashReader {
|
||||
}
|
||||
|
||||
impl EtagResolvable for HashReader {
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
!self.diskable_md5 && self.inner.is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
if self.diskable_md5 {
|
||||
return None;
|
||||
|
||||
@@ -201,10 +201,31 @@ where
|
||||
boxed_reader(WarpReader::new(reader))
|
||||
}
|
||||
|
||||
impl<T> EtagResolvable for std::io::Cursor<T> {}
|
||||
|
||||
impl EtagResolvable for &[u8] {}
|
||||
|
||||
impl<R> EtagResolvable for tokio::io::BufReader<R>
|
||||
where
|
||||
R: tokio::io::AsyncRead + EtagResolvable,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.get_ref().is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.get_mut().try_resolve_etag()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> EtagResolvable for Box<T>
|
||||
where
|
||||
T: EtagResolvable + ?Sized,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.as_ref().is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.as_mut().try_resolve_etag()
|
||||
}
|
||||
|
||||
@@ -596,9 +596,205 @@ pub struct PendingScannerHeal {
|
||||
pub last_admission_reason: String,
|
||||
}
|
||||
|
||||
/// One bucket's standing in its set's service order. The wait position, the
|
||||
/// quanta granted and the last certification are separate quantities: a bucket
|
||||
/// that keeps failing moves to the tail like any other and only adds debt.
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketService {
|
||||
#[serde(default)]
|
||||
pub waiting_since: u64,
|
||||
#[serde(default)]
|
||||
pub service_quanta: u64,
|
||||
#[serde(default)]
|
||||
pub last_certified: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub failure_debt: u64,
|
||||
#[serde(default)]
|
||||
pub incarnation: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
// Nested derives encode positionally; a map lets later fields stay optional.
|
||||
impl Serialize for DataUsageBucketService {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let field_count = 2
|
||||
+ usize::from(self.last_certified.is_some())
|
||||
+ usize::from(self.failure_debt != 0)
|
||||
+ usize::from(self.incarnation.is_some());
|
||||
let mut map = serializer.serialize_map(Some(field_count))?;
|
||||
map.serialize_entry("waiting_since", &self.waiting_since)?;
|
||||
map.serialize_entry("service_quanta", &self.service_quanta)?;
|
||||
if let Some(last_certified) = self.last_certified {
|
||||
map.serialize_entry("last_certified", &last_certified)?;
|
||||
}
|
||||
if self.failure_debt != 0 {
|
||||
map.serialize_entry("failure_debt", &self.failure_debt)?;
|
||||
}
|
||||
if let Some(incarnation) = self.incarnation {
|
||||
map.serialize_entry("incarnation", &incarnation)?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable first-in-first-out service order of one set, in generations: every
|
||||
/// bucket at or below `generation_floor` still waits for its turn and everything
|
||||
/// above it was served, so a new bucket queues behind waiters but ahead of
|
||||
/// buckets that already had a turn. It only orders work:
|
||||
/// no entry proves coverage, authorizes a skip or releases a responsibility,
|
||||
/// so losing it restarts fairness and nothing else. Like the set cache's bucket
|
||||
/// entries and incarnations, it holds one record per bucket of the inventory.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketServiceLedger {
|
||||
/// Movement epoch the order was built under; another epoch invalidates it.
|
||||
#[serde(default)]
|
||||
pub publication_epoch: u64,
|
||||
/// Leader that saved the order; an incomplete save stamps nothing else.
|
||||
#[serde(default)]
|
||||
pub leader_epoch: u64,
|
||||
#[serde(default)]
|
||||
pub next_sequence: u64,
|
||||
#[serde(default)]
|
||||
pub generation_floor: u64,
|
||||
#[serde(default)]
|
||||
pub buckets: HashMap<String, DataUsageBucketService>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageBucketServiceLedger {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut map = serializer.serialize_map(Some(5))?;
|
||||
map.serialize_entry("publication_epoch", &self.publication_epoch)?;
|
||||
map.serialize_entry("leader_epoch", &self.leader_epoch)?;
|
||||
map.serialize_entry("next_sequence", &self.next_sequence)?;
|
||||
map.serialize_entry("generation_floor", &self.generation_floor)?;
|
||||
map.serialize_entry("buckets", &self.buckets)?;
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageBucketServiceLedger {
|
||||
/// Rebuilds the order for this cycle's inventory. Decoded state that fails
|
||||
/// validation or belongs to another movement epoch starts a new order;
|
||||
/// a bucket whose incarnation changed is a new arrival behind all waiters.
|
||||
pub(crate) fn restore<'a>(
|
||||
persisted: Option<Self>,
|
||||
publication_epoch: u64,
|
||||
inventory: impl Iterator<Item = &'a str>,
|
||||
incarnations: Option<&HashMap<String, uuid::Uuid>>,
|
||||
) -> Self {
|
||||
let mut ledger = persisted
|
||||
.filter(|ledger| ledger.publication_epoch == publication_epoch && ledger.is_valid())
|
||||
.unwrap_or_else(|| Self {
|
||||
publication_epoch,
|
||||
..Default::default()
|
||||
});
|
||||
let previous = std::mem::take(&mut ledger.buckets);
|
||||
let mut arrivals = Vec::new();
|
||||
for bucket in inventory {
|
||||
let incarnation = incarnations.and_then(|current| current.get(bucket)).copied();
|
||||
match previous.get(bucket) {
|
||||
Some(service) if service.incarnation.is_none() || incarnation.is_none() || service.incarnation == incarnation => {
|
||||
ledger.buckets.insert(
|
||||
bucket.to_owned(),
|
||||
DataUsageBucketService {
|
||||
incarnation: incarnation.or(service.incarnation),
|
||||
..*service
|
||||
},
|
||||
);
|
||||
}
|
||||
_ => arrivals.push((bucket, incarnation)),
|
||||
}
|
||||
}
|
||||
if arrivals.is_empty() {
|
||||
let floor = ledger.generation_floor;
|
||||
if ledger.buckets.values().all(|service| service.waiting_since > floor) {
|
||||
ledger.generation_floor = ledger.next_sequence;
|
||||
}
|
||||
return ledger;
|
||||
}
|
||||
// Open one position between the waiters and the served buckets.
|
||||
let floor = ledger.generation_floor;
|
||||
ledger.generation_floor = floor.saturating_add(1);
|
||||
ledger.next_sequence = ledger.next_sequence.saturating_add(1);
|
||||
for service in ledger.buckets.values_mut().filter(|service| service.waiting_since > floor) {
|
||||
service.waiting_since = service.waiting_since.saturating_add(1);
|
||||
}
|
||||
for (bucket, incarnation) in arrivals {
|
||||
ledger.buckets.insert(
|
||||
bucket.to_owned(),
|
||||
DataUsageBucketService {
|
||||
waiting_since: ledger.generation_floor,
|
||||
incarnation,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
ledger
|
||||
}
|
||||
|
||||
fn is_valid(&self) -> bool {
|
||||
self.generation_floor <= self.next_sequence
|
||||
&& self
|
||||
.buckets
|
||||
.iter()
|
||||
.all(|(bucket, service)| !bucket.is_empty() && service.waiting_since <= self.next_sequence)
|
||||
}
|
||||
|
||||
/// Smaller positions have waited longer; an untracked bucket queues last.
|
||||
pub(crate) fn wait_position(&self, bucket: &str) -> u64 {
|
||||
self.buckets.get(bucket).map_or(u64::MAX, |service| service.waiting_since)
|
||||
}
|
||||
|
||||
/// Moves a bucket to the tail and returns its previous wait position.
|
||||
pub(crate) fn record_service(&mut self, bucket: &str) -> Option<u64> {
|
||||
let service = self.buckets.get_mut(bucket)?;
|
||||
self.next_sequence = self.next_sequence.saturating_add(1);
|
||||
let previous = std::mem::replace(&mut service.waiting_since, self.next_sequence);
|
||||
service.service_quanta = service.service_quanta.saturating_add(1);
|
||||
Some(previous)
|
||||
}
|
||||
|
||||
/// Undoes `record_service` for an admission a resource veto turned back.
|
||||
pub(crate) fn return_service(&mut self, bucket: &str, waiting_since: u64) {
|
||||
if let Some(service) = self.buckets.get_mut(bucket) {
|
||||
service.waiting_since = waiting_since;
|
||||
service.service_quanta = service.service_quanta.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_certified(&mut self, bucket: &str) {
|
||||
if let Some(service) = self.buckets.get_mut(bucket) {
|
||||
service.last_certified = Some(service.waiting_since);
|
||||
service.failure_debt = 0;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_failure(&mut self, bucket: &str) {
|
||||
if let Some(service) = self.buckets.get_mut(bucket) {
|
||||
service.failure_debt = service.failure_debt.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Buckets holding unpaid failures and the first of them by name.
|
||||
pub(crate) fn failure_debt(&self) -> (usize, Option<&str>) {
|
||||
let mut debtors = self
|
||||
.buckets
|
||||
.iter()
|
||||
.filter(|(_, service)| service.failure_debt > 0)
|
||||
.map(|(bucket, _)| bucket.as_str());
|
||||
let first = debtors.next();
|
||||
let (count, first) = debtors.fold((usize::from(first.is_some()), first), |(count, first), bucket| {
|
||||
(count + 1, first.min(Some(bucket)))
|
||||
});
|
||||
(count, first)
|
||||
}
|
||||
}
|
||||
|
||||
/// Data usage cache info
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageCacheInfo {
|
||||
/// In-memory only: this partial stopped on admission pressure, not a failure.
|
||||
#[serde(skip)]
|
||||
pub scan_resource_deferred: bool,
|
||||
pub name: String,
|
||||
pub next_cycle: u64,
|
||||
pub last_update: Option<SystemTime>,
|
||||
@@ -615,6 +811,11 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
/// Indexes of the other directories on the active walk path. A restart
|
||||
/// must replay every wide ancestor before it reaches the selected one, and
|
||||
/// recharging those replays would stop a fixed budget from ever advancing.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub scan_raw_enumeration_path_page_indexes: Vec<RawEnumerationPageIndex>,
|
||||
#[serde(default)]
|
||||
pub scan_identity: Option<DataUsageScanIdentity>,
|
||||
#[serde(default)]
|
||||
@@ -674,6 +875,9 @@ pub struct DataUsageCacheInfo {
|
||||
/// skipping an unselected bucket in a later scoped set scan.
|
||||
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
|
||||
pub scan_bucket_incarnations: HashMap<String, uuid::Uuid>,
|
||||
/// Bucket service order of a set cache; absent from bucket caches.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_bucket_service: Option<DataUsageBucketServiceLedger>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageCacheInfo {
|
||||
@@ -686,6 +890,7 @@ impl Serialize for DataUsageCacheInfo {
|
||||
let field_count = 16
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
|
||||
+ usize::from(!self.scan_raw_enumeration_path_page_indexes.is_empty())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_coverage_receipt.is_some())
|
||||
@@ -699,7 +904,8 @@ impl Serialize for DataUsageCacheInfo {
|
||||
+ usize::from(self.lkg_scan_plan_digest.is_some())
|
||||
+ usize::from(self.scan_execution_digest.is_some())
|
||||
+ usize::from(self.segment_invalidation_proof.is_some())
|
||||
+ usize::from(!self.scan_bucket_incarnations.is_empty());
|
||||
+ usize::from(!self.scan_bucket_incarnations.is_empty())
|
||||
+ usize::from(self.scan_bucket_service.is_some());
|
||||
let mut state = serializer.serialize_map(Some(field_count))?;
|
||||
state.serialize_entry("name", &self.name)?;
|
||||
state.serialize_entry("next_cycle", &self.next_cycle)?;
|
||||
@@ -717,6 +923,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(index) = &self.scan_raw_enumeration_page_index {
|
||||
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
|
||||
}
|
||||
if !self.scan_raw_enumeration_path_page_indexes.is_empty() {
|
||||
state.serialize_entry("scan_raw_enumeration_path_page_indexes", &self.scan_raw_enumeration_path_page_indexes)?;
|
||||
}
|
||||
if let Some(identity) = self.scan_identity {
|
||||
state.serialize_entry("scan_identity", &identity)?;
|
||||
}
|
||||
@@ -765,6 +974,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if !self.scan_bucket_incarnations.is_empty() {
|
||||
state.serialize_entry("scan_bucket_incarnations", &self.scan_bucket_incarnations)?;
|
||||
}
|
||||
if let Some(ledger) = &self.scan_bucket_service {
|
||||
state.serialize_entry("scan_bucket_service", ledger)?;
|
||||
}
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -902,6 +1114,30 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Identifies how far durable resume state has advanced. Discovery alone
|
||||
/// does not change it; only committed raw pages or coverage frontier do.
|
||||
pub(crate) fn durable_scan_position(&self) -> (Option<String>, Option<String>, u64) {
|
||||
let raw_entries = self
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.iter()
|
||||
.chain(&self.info.scan_raw_enumeration_path_page_indexes)
|
||||
.map(|index| match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => 0,
|
||||
RawEnumerationPageOwnerStatus::Building { indexed_entries, .. }
|
||||
| RawEnumerationPageOwnerStatus::Ready { indexed_entries, .. } => indexed_entries,
|
||||
})
|
||||
.fold(0, u64::saturating_add);
|
||||
(
|
||||
self.info.scan_resume_after.clone(),
|
||||
self.info
|
||||
.scan_coverage_receipt
|
||||
.as_ref()
|
||||
.map(|receipt| receipt.through.clone()),
|
||||
raw_entries,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn has_scan_position(&self) -> bool {
|
||||
self.info.scan_resume_after.is_some()
|
||||
|| self.info.scan_checkpoint.is_some()
|
||||
@@ -1044,7 +1280,14 @@ impl DataUsageCache {
|
||||
}
|
||||
if self.validated_raw_enumeration_page_index().is_none() {
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
}
|
||||
let path_indexes = std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes);
|
||||
self.info.scan_raw_enumeration_path_page_indexes = path_indexes
|
||||
.into_iter()
|
||||
.take(MAX_DATA_USAGE_CACHE_DEPTH)
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
.collect();
|
||||
let cursor_is_valid = !self.has_scan_position()
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
@@ -1087,6 +1330,7 @@ impl DataUsageCache {
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -1230,22 +1474,37 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub(crate) fn validated_raw_enumeration_page_index(&self) -> Option<&RawEnumerationPageIndex> {
|
||||
let index = self.info.scan_raw_enumeration_page_index.as_ref()?;
|
||||
self.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.as_ref()
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
}
|
||||
|
||||
/// The durable index of one directory on the saved walk path.
|
||||
pub(crate) fn validated_raw_enumeration_page_index_for_parent(&self, parent: &str) -> Option<&RawEnumerationPageIndex> {
|
||||
self.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.iter()
|
||||
.chain(&self.info.scan_raw_enumeration_path_page_indexes)
|
||||
.find(|index| index.parent() == Some(parent))
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
}
|
||||
|
||||
fn raw_enumeration_page_index_is_valid(&self, index: &RawEnumerationPageIndex) -> bool {
|
||||
if self.info.scan_progress.is_none()
|
||||
|| !self.info.scan_identity.is_some_and(|identity| identity.is_valid())
|
||||
|| self.info.source.is_none()
|
||||
|| index.committed_entries().is_err()
|
||||
|| index.indexed_entries().is_err()
|
||||
{
|
||||
return None;
|
||||
return false;
|
||||
}
|
||||
let parent = match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => return None,
|
||||
match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => false,
|
||||
RawEnumerationPageOwnerStatus::Building { parent, .. } | RawEnumerationPageOwnerStatus::Ready { parent, .. } => {
|
||||
parent
|
||||
path_is_in_bucket_scope(&self.info.name, &parent)
|
||||
}
|
||||
};
|
||||
path_is_in_bucket_scope(&self.info.name, &parent).then_some(index)
|
||||
}
|
||||
}
|
||||
|
||||
/// Seal only the frontier supplied by completed traversal, never a restored cursor.
|
||||
|
||||
@@ -57,6 +57,66 @@ fn scoped_scan_coverage_metadata_preserves_map_compatibility() {
|
||||
assert_eq!(decoded.scan_coverage_digest, Some(TEST_PLAN_DIGEST));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_path_indexes_preserve_map_compatibility_and_validation() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct LegacyInfo {
|
||||
name: String,
|
||||
scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
}
|
||||
let index_for = |parent: &str| RawEnumerationPageIndex::new(parent, 8).expect("fixture index");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
scan_progress: Some(DataUsageScanProgress {
|
||||
started_plan: TEST_PLAN_DIGEST,
|
||||
requested_plan: TEST_PLAN_DIGEST,
|
||||
}),
|
||||
scan_identity: Some(DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(1),
|
||||
set_layout: DataUsageScanPlanDigest([2; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
}),
|
||||
source: Some(DataUsageCacheSource::new(0, 0)),
|
||||
scan_raw_enumeration_page_index: Some(index_for("bucket/wide")),
|
||||
scan_raw_enumeration_path_page_indexes: vec![index_for("bucket"), index_for("other-bucket/wide")],
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
};
|
||||
assert!(cache.validated_raw_enumeration_page_index_for_parent("bucket").is_some());
|
||||
assert!(
|
||||
cache
|
||||
.validated_raw_enumeration_page_index_for_parent("other-bucket/wide")
|
||||
.is_none(),
|
||||
"a saved path index must stay inside its bucket"
|
||||
);
|
||||
|
||||
let encoded = rmp_serde::to_vec(&cache.info).expect("path indexes should remain map encoded");
|
||||
let legacy: LegacyInfo = rmp_serde::from_slice(&encoded).expect("old map readers should ignore the path indexes");
|
||||
assert_eq!(legacy.name, "bucket");
|
||||
assert_eq!(
|
||||
legacy
|
||||
.scan_raw_enumeration_page_index
|
||||
.and_then(|index| index.parent().map(str::to_owned)),
|
||||
Some("bucket/wide".to_string())
|
||||
);
|
||||
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&encoded).expect("new reader should restore the path indexes");
|
||||
assert_eq!(decoded.scan_raw_enumeration_path_page_indexes.len(), 2);
|
||||
|
||||
cache.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
let encoded = rmp_serde::to_vec(&cache.info).expect("empty path indexes should encode");
|
||||
assert!(
|
||||
!encoded
|
||||
.windows(b"scan_raw_enumeration_path_page_indexes".len())
|
||||
.any(|window| window == b"scan_raw_enumeration_path_page_indexes"),
|
||||
"the default must stay absent from the map"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct CachePutRecord {
|
||||
object: String,
|
||||
|
||||
@@ -80,6 +80,7 @@ impl RemoteScannerBudget {
|
||||
max_duration: self.max_duration_ms.map(Duration::from_millis),
|
||||
max_objects: self.max_objects,
|
||||
max_directories: self.max_directories,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,6 +743,7 @@ pub(crate) fn lookup_scanner_runtime_config(
|
||||
max_duration: cycle_max_duration.filter(|duration| !duration.is_zero()),
|
||||
max_objects: cycle_max_objects,
|
||||
max_directories: cycle_max_directories,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (bitrot_cycle, bitrot_cycle_source) = if let Some(value) = rustfs_utils::get_env_opt_str(ENV_SCANNER_BITROT_CYCLE_SECS) {
|
||||
@@ -1171,6 +1172,7 @@ mod tests {
|
||||
max_duration: Some(Duration::from_secs(1800)),
|
||||
max_objects: Some(1000),
|
||||
max_directories: Some(8000),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -474,14 +474,17 @@ fn scan_cycle_partial_reason(reason: Option<ScannerCycleBudgetReason>) -> ScanCy
|
||||
match reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => ScanCyclePartialReason::Runtime,
|
||||
Some(ScannerCycleBudgetReason::Objects) => ScanCyclePartialReason::Objects,
|
||||
Some(ScannerCycleBudgetReason::Directories) => ScanCyclePartialReason::Directories,
|
||||
// Raw enumeration is directory work; the reason set is a stable metric label.
|
||||
Some(ScannerCycleBudgetReason::Directories | ScannerCycleBudgetReason::Entries) => ScanCyclePartialReason::Directories,
|
||||
None => ScanCyclePartialReason::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_cycle_partial_source(reason: Option<ScannerCycleBudgetReason>) -> Option<ScannerWorkSource> {
|
||||
match reason {
|
||||
Some(ScannerCycleBudgetReason::Objects | ScannerCycleBudgetReason::Directories) => Some(ScannerWorkSource::Usage),
|
||||
Some(ScannerCycleBudgetReason::Objects | ScannerCycleBudgetReason::Directories | ScannerCycleBudgetReason::Entries) => {
|
||||
Some(ScannerWorkSource::Usage)
|
||||
}
|
||||
Some(ScannerCycleBudgetReason::Runtime) | None => None,
|
||||
}
|
||||
}
|
||||
@@ -1059,6 +1062,31 @@ fn prepare_cycle_for_usage_floor_bootstrap(
|
||||
}
|
||||
}
|
||||
|
||||
/// Overwrite an unreadable usage floor with a bootstrap marker using the
|
||||
/// file's current revision for CAS. A successful publish leaves a
|
||||
/// `BootstrapPending` floor that the normal load path picks up on retry.
|
||||
/// Returns `Err` when the floor is not even readable (transient I/O) so the
|
||||
/// caller keeps the supervisor-retry behavior instead of rebuilding.
|
||||
async fn rebuild_unreadable_usage_floor(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> Result<(), ScannerError> {
|
||||
let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return Err(ScannerError::Other("scanner usage floor rebuild is blocked by data movement".to_string()));
|
||||
};
|
||||
let (_data, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("scanner usage floor revision unreadable for rebuild: {err}")))?;
|
||||
publish_scanner_usage_bootstrap_primary(
|
||||
storeapi,
|
||||
&revision,
|
||||
expected_epoch,
|
||||
None,
|
||||
ScannerUsageBootstrapPublishContext::Recovery,
|
||||
|| true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
init_data_scanner_with_storage(ctx, storeapi, ScannerRuntimeGuard::new()).await;
|
||||
}
|
||||
@@ -3087,6 +3115,27 @@ where
|
||||
Err(err) => usage_floor_result = Err(err),
|
||||
}
|
||||
}
|
||||
// backlog#2918: after dual-corruption recovery has been attempted an
|
||||
// unreadable floor is no longer a quota proof. Report the failure, then
|
||||
// rebuild through a CAS overwrite before giving up. Transient I/O errors
|
||||
// fail the rebuild and fall through to the supervisor-retry stop below.
|
||||
if usage_floor_result.is_err() {
|
||||
let error = usage_floor_result.as_ref().unwrap_err().to_string();
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "usage_floor_load_failed",
|
||||
error = %error,
|
||||
"Scanner usage floor is unreadable after recovery; attempting rebuild"
|
||||
);
|
||||
record_scanner_usage_floor_failure(error);
|
||||
if rebuild_unreadable_usage_floor(storeapi.clone()).await.is_ok() {
|
||||
usage_floor_result = persisted_usage_floor_for_startup(storeapi.clone(), true).await;
|
||||
}
|
||||
}
|
||||
let (usage_floor, usage_floor_startup) = match usage_floor_result {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
@@ -3099,7 +3148,7 @@ where
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "usage_floor_load_failed",
|
||||
error = %err,
|
||||
"Scanner stopped because the persisted usage floor could not be loaded"
|
||||
"Scanner usage floor could not be loaded or rebuilt; stopping leader iteration for supervisor retry"
|
||||
);
|
||||
record_scanner_usage_floor_failure(error.clone());
|
||||
global_metrics().set_cycle(None).await;
|
||||
|
||||
@@ -41,20 +41,6 @@ fn reset_recovery_status() {
|
||||
.expect("test must be able to reset the local recovery observation");
|
||||
}
|
||||
|
||||
async fn wait_for_recovery_attempt(attempt: u32) -> ScannerCycleRecoveryStatus {
|
||||
tokio::time::timeout(WAIT_BUDGET, async {
|
||||
loop {
|
||||
let status = scanner_cycle_recovery_status();
|
||||
if status.retry_count >= u64::from(attempt) {
|
||||
return status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("recovery attempt {attempt} was never recorded: {:?}", scanner_cycle_recovery_status()))
|
||||
}
|
||||
|
||||
async fn wait_for_recovery_state(state: &str) -> ScannerCycleRecoveryStatus {
|
||||
tokio::time::timeout(WAIT_BUDGET, async {
|
||||
loop {
|
||||
@@ -130,20 +116,19 @@ fn started_rebalance() -> EcstoreRebalanceMeta {
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8670: a usage floor that is present but untrusted must not
|
||||
/// drive the supervisor into a terminal `paused, retryable=false` observation.
|
||||
/// Every attempt stays retryable with a bounded backoff, and repairing the
|
||||
/// floor recovers the scanner without a process restart.
|
||||
/// rustfs/rustfs#8670 + backlog#2918: a usage floor that is present but
|
||||
/// untrusted must not stop the scanner. The unreadable floor is reported and
|
||||
/// rebuilt through a CAS overwrite so the scanner recovers without a process
|
||||
/// restart or a terminal `paused, retryable=false` observation.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn usage_floor_load_failure_backs_off_and_recovers_without_restart() {
|
||||
async fn usage_floor_load_failure_rebuilds_and_recovers_without_restart() {
|
||||
temp_env::async_with_vars(SUPERVISOR_ENV, async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
reset_recovery_status();
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
// Present but without a baseline identity: the shape #8670 reported,
|
||||
// whose error points the operator at the usage-state reset.
|
||||
// Present but without a baseline identity: the shape #8670 reported.
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
@@ -156,41 +141,11 @@ async fn usage_floor_load_failure_backs_off_and_recovers_without_restart() {
|
||||
let _cancel_on_drop = ctx.clone().drop_guard();
|
||||
init_data_scanner_with_storage(ctx.clone(), store.clone(), ScannerRuntimeGuard::new()).await;
|
||||
|
||||
let attempts = MAX_SCANNER_CYCLE_RECOVERY_RETRIES + 2;
|
||||
let mut first_detected = None;
|
||||
for attempt in 1..=attempts {
|
||||
let status = wait_for_recovery_attempt(attempt).await;
|
||||
assert_eq!(status.classification.as_deref(), Some("usage_floor_load_failed"), "attempt {attempt}");
|
||||
assert!(status.retryable, "attempt {attempt} must stay retryable: {status:?}");
|
||||
assert_ne!(status.state, "paused", "attempt {attempt} must not report a terminal pause");
|
||||
if attempt >= MAX_SCANNER_CYCLE_RECOVERY_RETRIES {
|
||||
assert_eq!(status.state, "backoff", "attempt {attempt}: {status:?}");
|
||||
} else {
|
||||
assert_eq!(status.state, "usage_floor_load_failed", "attempt {attempt}: {status:?}");
|
||||
}
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("usage-state/reset") && reason.contains("next automatic attempt")),
|
||||
"the operator hint and the next attempt must both be visible: {status:?}"
|
||||
);
|
||||
assert!(status.next_attempt_at_unix_secs.is_some(), "attempt {attempt}: {status:?}");
|
||||
match first_detected {
|
||||
None => first_detected = status.first_detected_at_unix_secs,
|
||||
Some(first) => assert_eq!(status.first_detected_at_unix_secs, Some(first), "attempt {attempt}"),
|
||||
}
|
||||
if attempt == attempts {
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), valid_usage_floor(0, 0))
|
||||
.await
|
||||
.expect("repaired usage floor should persist");
|
||||
}
|
||||
notify_scanner_cycle_recovery_wake();
|
||||
}
|
||||
|
||||
// backlog#2918: the unreadable floor is rebuilt automatically; the
|
||||
// scanner must reach healthy without a manual repair or terminal pause.
|
||||
let healthy = wait_for_recovery_state("healthy").await;
|
||||
assert_eq!(healthy.retry_count, 0, "a successful attempt clears the retry budget: {healthy:?}");
|
||||
assert!(!healthy.retryable);
|
||||
assert!(!healthy.retryable, "rebuild must clear the retry budget: {healthy:?}");
|
||||
assert_ne!(healthy.state, "paused", "rebuild must not report a terminal pause");
|
||||
wait_for_persisted_cycle(&store, |cycle, _| cycle.next >= 1).await;
|
||||
assert_eq!(scanner_cycle_schedule_status().execution_role, "leader");
|
||||
stop_supervisor(ctx).await;
|
||||
|
||||
@@ -24,13 +24,51 @@ const BUDGET_REASON_NONE: u8 = 0;
|
||||
const BUDGET_REASON_RUNTIME: u8 = 1;
|
||||
const BUDGET_REASON_OBJECTS: u8 = 2;
|
||||
const BUDGET_REASON_DIRECTORIES: u8 = 3;
|
||||
const BUDGET_REASON_ENTRIES: u8 = 4;
|
||||
const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128;
|
||||
/// One raw-enumeration page, so a directory stop always commits durable work.
|
||||
const DIRECTORY_ENTRY_BUDGET_FLOOR: u64 = 128;
|
||||
/// Directory entries one object or directory budget unit pays for. A forward
|
||||
/// sweep classifies in sorted order and so needs a whole listing first; a
|
||||
/// directory a few times wider than the budget must still reach objects in
|
||||
/// its first round, while a far wider one builds its index across rounds.
|
||||
const DIRECTORY_ENTRIES_PER_COUNT_UNIT: u64 = 16;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerCycleBudgetConfig {
|
||||
pub max_duration: Option<Duration>,
|
||||
pub max_objects: Option<u64>,
|
||||
pub max_directories: Option<u64>,
|
||||
/// Raw directory entries first observed under this budget. Entries replayed
|
||||
/// from a validated page index are not charged.
|
||||
pub max_entries: Option<u64>,
|
||||
/// Cumulative name bytes of the charged raw entries.
|
||||
pub max_entry_name_bytes: Option<u64>,
|
||||
/// One bucket's share. `None` lets a bucket consume the whole budget.
|
||||
pub bucket_quantum: Option<ScannerBucketQuantum>,
|
||||
}
|
||||
|
||||
/// Limits of one bucket admission, charged to the owning cycle budget as well.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerBucketQuantum {
|
||||
pub max_duration: Option<Duration>,
|
||||
pub max_objects: Option<u64>,
|
||||
pub max_directories: Option<u64>,
|
||||
pub max_entries: Option<u64>,
|
||||
pub max_entry_name_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
impl From<ScannerBucketQuantum> for ScannerCycleBudgetConfig {
|
||||
fn from(quantum: ScannerBucketQuantum) -> Self {
|
||||
Self {
|
||||
max_duration: quantum.max_duration,
|
||||
max_objects: quantum.max_objects,
|
||||
max_directories: quantum.max_directories,
|
||||
max_entries: quantum.max_entries,
|
||||
max_entry_name_bytes: quantum.max_entry_name_bytes,
|
||||
bucket_quantum: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -38,6 +76,7 @@ pub(crate) enum ScannerCycleBudgetReason {
|
||||
Runtime,
|
||||
Objects,
|
||||
Directories,
|
||||
Entries,
|
||||
}
|
||||
|
||||
impl ScannerCycleBudgetReason {
|
||||
@@ -46,6 +85,7 @@ impl ScannerCycleBudgetReason {
|
||||
Self::Runtime => BUDGET_REASON_RUNTIME,
|
||||
Self::Objects => BUDGET_REASON_OBJECTS,
|
||||
Self::Directories => BUDGET_REASON_DIRECTORIES,
|
||||
Self::Entries => BUDGET_REASON_ENTRIES,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,12 +94,15 @@ impl ScannerCycleBudgetReason {
|
||||
BUDGET_REASON_RUNTIME => Some(Self::Runtime),
|
||||
BUDGET_REASON_OBJECTS => Some(Self::Objects),
|
||||
BUDGET_REASON_DIRECTORIES => Some(Self::Directories),
|
||||
BUDGET_REASON_ENTRIES => Some(Self::Entries),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ScannerCycleBudget {
|
||||
/// The cycle budget a bucket quantum also charges.
|
||||
parent: Option<Arc<ScannerCycleBudget>>,
|
||||
token: CancellationToken,
|
||||
reason: Arc<AtomicU8>,
|
||||
started_at: Instant,
|
||||
@@ -67,11 +110,17 @@ pub struct ScannerCycleBudget {
|
||||
max_duration: Option<Duration>,
|
||||
max_objects: Option<u64>,
|
||||
max_directories: Option<u64>,
|
||||
max_entries: Option<u64>,
|
||||
max_entry_name_bytes: Option<u64>,
|
||||
directory_entry_limit: Option<u64>,
|
||||
bucket_quantum: Option<ScannerBucketQuantum>,
|
||||
track_progress: bool,
|
||||
track_unbounded_counts: bool,
|
||||
objects_scanned: AtomicU64,
|
||||
directories_started: AtomicU64,
|
||||
entries_visited: AtomicU64,
|
||||
entries_charged: AtomicU64,
|
||||
entry_name_bytes_charged: AtomicU64,
|
||||
last_progress_millis: AtomicU64,
|
||||
cycle_state_persisted: AtomicBool,
|
||||
remote_scan_progress: Arc<RemoteScanProgress>,
|
||||
@@ -80,16 +129,16 @@ pub struct ScannerCycleBudget {
|
||||
impl ScannerCycleBudget {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
Self::new_inner(parent, config, false, false)
|
||||
Self::new_inner(parent, config, false, false, None)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
Self::new_inner(parent, config, true, true)
|
||||
Self::new_inner(parent, config, true, true, None)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
let track_progress = config.max_duration.is_some();
|
||||
Self::new_inner(parent, config, track_progress, false)
|
||||
Self::new_inner(parent, config, track_progress, false, None)
|
||||
}
|
||||
|
||||
fn new_inner(
|
||||
@@ -97,6 +146,7 @@ impl ScannerCycleBudget {
|
||||
config: ScannerCycleBudgetConfig,
|
||||
track_progress: bool,
|
||||
track_unbounded_counts: bool,
|
||||
cycle_budget: Option<Arc<ScannerCycleBudget>>,
|
||||
) -> Arc<Self> {
|
||||
let token = parent.child_token();
|
||||
let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
|
||||
@@ -124,7 +174,11 @@ impl ScannerCycleBudget {
|
||||
});
|
||||
}
|
||||
|
||||
let remote_scan_progress = cycle_budget
|
||||
.as_ref()
|
||||
.map_or_else(|| Arc::new(RemoteScanProgress::default()), |budget| budget.remote_scan_progress());
|
||||
Arc::new(Self {
|
||||
parent: cycle_budget,
|
||||
token,
|
||||
reason,
|
||||
started_at,
|
||||
@@ -132,14 +186,20 @@ impl ScannerCycleBudget {
|
||||
max_duration: config.max_duration,
|
||||
max_objects: config.max_objects,
|
||||
max_directories: config.max_directories,
|
||||
max_entries: config.max_entries,
|
||||
max_entry_name_bytes: config.max_entry_name_bytes,
|
||||
directory_entry_limit: directory_entry_limit(&config),
|
||||
bucket_quantum: config.bucket_quantum,
|
||||
track_progress,
|
||||
track_unbounded_counts,
|
||||
objects_scanned: AtomicU64::new(0),
|
||||
directories_started: AtomicU64::new(0),
|
||||
entries_visited: AtomicU64::new(0),
|
||||
entries_charged: AtomicU64::new(0),
|
||||
entry_name_bytes_charged: AtomicU64::new(0),
|
||||
last_progress_millis: AtomicU64::new(0),
|
||||
cycle_state_persisted: AtomicBool::new(false),
|
||||
remote_scan_progress: Arc::new(RemoteScanProgress::default()),
|
||||
remote_scan_progress,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -147,12 +207,31 @@ impl ScannerCycleBudget {
|
||||
self.token.clone()
|
||||
}
|
||||
|
||||
/// Budget and cancellation scope of one bucket scan. Without a quantum the
|
||||
/// bucket shares this budget; a quantum stops only its own scan token.
|
||||
pub(crate) fn for_bucket(self: &Arc<Self>, ctx: &CancellationToken) -> (Arc<Self>, CancellationToken) {
|
||||
let Some(quantum) = self.bucket_quantum else {
|
||||
return (self.clone(), ctx.child_token());
|
||||
};
|
||||
let budget = Self::new_inner(ctx, quantum.into(), self.track_progress, self.track_unbounded_counts, Some(self.clone()));
|
||||
let token = budget.token();
|
||||
(budget, token)
|
||||
}
|
||||
|
||||
/// True when only this bucket's share ended and the cycle can continue.
|
||||
pub(crate) fn quantum_exhausted(&self) -> bool {
|
||||
self.parent.as_ref().is_some_and(|parent| !parent.budget_elapsed())
|
||||
&& self.reason.load(Ordering::Relaxed) != BUDGET_REASON_NONE
|
||||
}
|
||||
|
||||
pub(crate) fn budget_elapsed(&self) -> bool {
|
||||
self.reason.load(Ordering::Relaxed) != BUDGET_REASON_NONE
|
||||
|| self.parent.as_ref().is_some_and(|parent| parent.budget_elapsed())
|
||||
}
|
||||
|
||||
pub(crate) fn reason(&self) -> Option<ScannerCycleBudgetReason> {
|
||||
ScannerCycleBudgetReason::from_code(self.reason.load(Ordering::Relaxed))
|
||||
.or_else(|| self.parent.as_ref().and_then(|parent| parent.reason()))
|
||||
}
|
||||
|
||||
pub(crate) fn max_duration(&self) -> Option<Duration> {
|
||||
@@ -176,7 +255,10 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn requires_serial_progress_accounting(&self) -> bool {
|
||||
self.max_objects.is_some() || self.max_directories.is_some()
|
||||
self.max_objects.is_some()
|
||||
|| self.max_directories.is_some()
|
||||
|| self.max_entries.is_some()
|
||||
|| self.max_entry_name_bytes.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn remaining_config(&self) -> ScannerCycleBudgetConfig {
|
||||
@@ -195,6 +277,13 @@ impl ScannerCycleBudget {
|
||||
max_directories: self
|
||||
.max_directories
|
||||
.map(|max| max.saturating_sub(self.directories_started.load(Ordering::Relaxed))),
|
||||
max_entries: self
|
||||
.max_entries
|
||||
.map(|max| max.saturating_sub(self.entries_charged.load(Ordering::Relaxed))),
|
||||
max_entry_name_bytes: self
|
||||
.max_entry_name_bytes
|
||||
.map(|max| max.saturating_sub(self.entry_name_bytes_charged.load(Ordering::Relaxed))),
|
||||
bucket_quantum: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,6 +323,9 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
|
||||
if let Some(parent) = &self.parent {
|
||||
parent.record_entries_visited(entries_visited);
|
||||
}
|
||||
if self.track_progress {
|
||||
let entries = saturating_fetch_add(&self.entries_visited, entries_visited);
|
||||
self.record_progress_sample(entries);
|
||||
@@ -245,7 +337,11 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
|
||||
self.remote_scan_progress.record(objects_scanned, directories_started);
|
||||
// A quantum shares the cycle's progress record; its parent writes it.
|
||||
match &self.parent {
|
||||
Some(parent) => parent.record_remote_progress(objects_scanned, directories_started),
|
||||
None => self.remote_scan_progress.record(objects_scanned, directories_started),
|
||||
}
|
||||
if self.track_progress || self.max_objects.is_some() {
|
||||
let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
|
||||
if self.track_progress {
|
||||
@@ -271,6 +367,9 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_after_unreported_remote_progress(&self) {
|
||||
if let Some(parent) = &self.parent {
|
||||
parent.cancel_after_unreported_remote_progress();
|
||||
}
|
||||
if self.max_objects.is_some() {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Objects);
|
||||
} else if self.max_directories.is_some() {
|
||||
@@ -279,6 +378,11 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn try_start_directory(&self) -> bool {
|
||||
let parent_allows = self.parent.as_ref().is_none_or(|parent| parent.try_start_directory());
|
||||
self.try_start_own_directory() && parent_allows
|
||||
}
|
||||
|
||||
fn try_start_own_directory(&self) -> bool {
|
||||
if self.max_directories.is_none() && !self.track_unbounded_counts {
|
||||
return true;
|
||||
}
|
||||
@@ -298,7 +402,35 @@ impl ScannerCycleBudget {
|
||||
true
|
||||
}
|
||||
|
||||
/// Charges one raw entry first observed in this scan. `directory_entries`
|
||||
/// counts the charged entries of the current directory: a count budget is
|
||||
/// a total work limit, so one listing cannot read past it.
|
||||
///
|
||||
/// Returns `false` after charging the entry that exhausts a limit; the
|
||||
/// caller stops with the entry already recorded in its durable index.
|
||||
pub(crate) fn try_charge_raw_entry(&self, name_bytes: u64, directory_entries: u64) -> bool {
|
||||
let parent_allows = self
|
||||
.parent
|
||||
.as_ref()
|
||||
.is_none_or(|parent| parent.try_charge_raw_entry(name_bytes, directory_entries));
|
||||
let entries = self
|
||||
.max_entries
|
||||
.is_some_and(|max| saturating_fetch_add(&self.entries_charged, 1) >= max);
|
||||
let name_bytes = self
|
||||
.max_entry_name_bytes
|
||||
.is_some_and(|max| saturating_fetch_add(&self.entry_name_bytes_charged, name_bytes) >= max);
|
||||
let directory = self.directory_entry_limit.is_some_and(|limit| directory_entries >= limit);
|
||||
if entries || name_bytes || directory {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Entries);
|
||||
return false;
|
||||
}
|
||||
parent_allows
|
||||
}
|
||||
|
||||
pub(crate) fn record_object_scanned(&self) {
|
||||
if let Some(parent) = &self.parent {
|
||||
parent.record_object_scanned();
|
||||
}
|
||||
if self.max_objects.is_none() && !self.track_unbounded_counts {
|
||||
return;
|
||||
}
|
||||
@@ -337,6 +469,21 @@ fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_entry_limit(config: &ScannerCycleBudgetConfig) -> Option<u64> {
|
||||
if config.max_objects.is_none() && config.max_directories.is_none() {
|
||||
return None;
|
||||
}
|
||||
let total = config
|
||||
.max_objects
|
||||
.unwrap_or(0)
|
||||
.saturating_add(config.max_directories.unwrap_or(0));
|
||||
Some(
|
||||
total
|
||||
.saturating_mul(DIRECTORY_ENTRIES_PER_COUNT_UNIT)
|
||||
.max(DIRECTORY_ENTRY_BUDGET_FLOOR),
|
||||
)
|
||||
}
|
||||
|
||||
fn directory_budget_exhausted(directories: u64, max_directories: u64) -> bool {
|
||||
// Saturation hides a remote max+1 update when the configured limit is the
|
||||
// largest representable counter. Treat that boundary as exhausted rather
|
||||
@@ -354,6 +501,127 @@ impl Drop for ScannerCycleBudget {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_budget_limits_entries_and_name_bytes_with_one_reason() {
|
||||
let parent = CancellationToken::new();
|
||||
let by_entries = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_entries: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(by_entries.try_charge_raw_entry(1, 1));
|
||||
assert!(!by_entries.try_charge_raw_entry(1, 2));
|
||||
assert_eq!(by_entries.reason(), Some(ScannerCycleBudgetReason::Entries));
|
||||
assert!(by_entries.token().is_cancelled());
|
||||
|
||||
let by_bytes = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_entry_name_bytes: Some(10),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(by_bytes.try_charge_raw_entry(9, 1));
|
||||
assert!(!by_bytes.try_charge_raw_entry(1, 2));
|
||||
assert_eq!(by_bytes.reason(), Some(ScannerCycleBudgetReason::Entries));
|
||||
assert!(!parent.is_cancelled(), "a budget stop must not cancel its parent token");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_budget_derives_a_directory_limit_only_from_count_budgets() {
|
||||
let parent = CancellationToken::new();
|
||||
let runtime_only = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default());
|
||||
assert!(runtime_only.try_charge_raw_entry(1, u64::MAX));
|
||||
|
||||
let small = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(small.try_charge_raw_entry(1, DIRECTORY_ENTRY_BUDGET_FLOOR - 1));
|
||||
assert!(!small.try_charge_raw_entry(1, DIRECTORY_ENTRY_BUDGET_FLOOR));
|
||||
|
||||
let large = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1000),
|
||||
max_directories: Some(24),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(large.try_charge_raw_entry(1, 1024 * DIRECTORY_ENTRIES_PER_COUNT_UNIT - 1));
|
||||
assert!(!large.try_charge_raw_entry(1, 1024 * DIRECTORY_ENTRIES_PER_COUNT_UNIT));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_bucket_share_stops_only_its_own_scan() {
|
||||
let ctx = CancellationToken::new();
|
||||
let cycle = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(3),
|
||||
bucket_quantum: Some(ScannerBucketQuantum {
|
||||
max_objects: Some(2),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (first, first_token) = cycle.for_bucket(&cycle.token());
|
||||
first.record_object_scanned();
|
||||
first.record_object_scanned();
|
||||
assert!(first.quantum_exhausted());
|
||||
assert!(first_token.is_cancelled());
|
||||
assert_eq!(first.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
assert!(!cycle.budget_elapsed(), "a bucket share leaves the cycle running");
|
||||
assert!(!cycle.token().is_cancelled());
|
||||
assert_eq!(cycle.progress().0, 2, "bucket work is charged to the cycle");
|
||||
|
||||
let (second, second_token) = cycle.for_bucket(&cycle.token());
|
||||
assert!(!second.budget_elapsed());
|
||||
second.record_object_scanned();
|
||||
assert!(cycle.budget_elapsed(), "the total budget still bounds every share");
|
||||
assert!(second_token.is_cancelled());
|
||||
assert!(second.budget_elapsed());
|
||||
assert!(!second.quantum_exhausted(), "a cycle stop is not a quantum handoff");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_bucket_share_runtime_expires_independently() {
|
||||
let ctx = CancellationToken::new();
|
||||
let cycle = ScannerCycleBudget::new(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
bucket_quantum: Some(ScannerBucketQuantum {
|
||||
max_duration: Some(Duration::ZERO),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (bucket, token) = cycle.for_bucket(&cycle.token());
|
||||
tokio::time::timeout(Duration::from_secs(5), token.cancelled())
|
||||
.await
|
||||
.expect("quantum deadline should cancel the bucket scan");
|
||||
assert_eq!(bucket.reason(), Some(ScannerCycleBudgetReason::Runtime));
|
||||
assert!(!cycle.budget_elapsed());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_without_a_quantum_shares_the_cycle_budget() {
|
||||
let ctx = CancellationToken::new();
|
||||
let cycle = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let (bucket, token) = cycle.for_bucket(&cycle.token());
|
||||
assert!(Arc::ptr_eq(&bucket, &cycle));
|
||||
cycle.cancel_for_runtime();
|
||||
assert!(token.is_cancelled());
|
||||
assert!(!bucket.quantum_exhausted());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_budget_cancels_child_token() {
|
||||
let parent = CancellationToken::new();
|
||||
@@ -441,6 +709,7 @@ mod tests {
|
||||
max_duration: Some(Duration::from_secs(60)),
|
||||
max_objects: Some(10),
|
||||
max_directories: Some(5),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -498,7 +498,10 @@ fn checkpoint_reason_from_budget(reason: Option<ScannerCycleBudgetReason>) -> Da
|
||||
match reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => DataUsageScanCheckpointReason::Runtime,
|
||||
Some(ScannerCycleBudgetReason::Objects) => DataUsageScanCheckpointReason::Objects,
|
||||
Some(ScannerCycleBudgetReason::Directories) => DataUsageScanCheckpointReason::Directories,
|
||||
// This enum is persisted and older readers reject unknown variants.
|
||||
Some(ScannerCycleBudgetReason::Directories | ScannerCycleBudgetReason::Entries) => {
|
||||
DataUsageScanCheckpointReason::Directories
|
||||
}
|
||||
None => DataUsageScanCheckpointReason::Unknown,
|
||||
}
|
||||
}
|
||||
@@ -1302,9 +1305,11 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) {
|
||||
/// Records one raw entry. Returns whether a durable index holds it, which
|
||||
/// is what makes stopping at this entry recoverable.
|
||||
fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) -> bool {
|
||||
if self.old_cache.info.scan_progress.is_none() {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
if let Some(position) = self
|
||||
.raw_enumeration_progress
|
||||
@@ -1317,18 +1322,14 @@ impl FolderScanner {
|
||||
self.raw_enumeration_progress
|
||||
.push(RawEnumerationProgress::new(parent, page_index));
|
||||
}
|
||||
if let Some(progress) = self.raw_enumeration_progress.last_mut() {
|
||||
self.raw_enumeration_progress.last_mut().is_some_and(|progress| {
|
||||
progress.record_entry(entry);
|
||||
}
|
||||
progress.page_index.is_some()
|
||||
})
|
||||
}
|
||||
|
||||
fn raw_enumeration_page_index_for_parent(&self, parent: &str) -> Option<&RawEnumerationPageIndex> {
|
||||
self.old_cache
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.as_ref()
|
||||
.filter(|index| index.parent() == Some(parent))?;
|
||||
self.old_cache.validated_raw_enumeration_page_index()
|
||||
self.old_cache.validated_raw_enumeration_page_index_for_parent(parent)
|
||||
}
|
||||
|
||||
fn raw_enumeration_committed_entry_oracle(&self, parent: &str) -> HashSet<String> {
|
||||
@@ -1372,23 +1373,36 @@ impl FolderScanner {
|
||||
if self.raw_enumeration_progress.is_empty() {
|
||||
return (None, None);
|
||||
}
|
||||
let progress_index = self
|
||||
.raw_enumeration_progress
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
let progress_index = self.selected_raw_enumeration_progress().unwrap_or(0);
|
||||
let progress = self.raw_enumeration_progress.swap_remove(progress_index);
|
||||
self.raw_enumeration_progress.clear();
|
||||
(progress.cursor(), progress.page_index())
|
||||
}
|
||||
|
||||
fn peek_raw_enumeration_resume_state(&self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
fn selected_raw_enumeration_progress(&self) -> Option<usize> {
|
||||
self.raw_enumeration_progress
|
||||
.iter()
|
||||
.max_by_key(|progress| progress.checkpointable_entry_count())
|
||||
.map_or((None, None), |progress| (progress.cursor(), progress.page_index()))
|
||||
.enumerate()
|
||||
.max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
|
||||
.map(|(index, _)| index)
|
||||
}
|
||||
|
||||
/// The walk path except the directory the resume state selects.
|
||||
fn raw_enumeration_path_page_indexes(&self) -> Vec<RawEnumerationPageIndex> {
|
||||
let selected = self.selected_raw_enumeration_progress();
|
||||
self.raw_enumeration_progress
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(index, _)| Some(*index) != selected)
|
||||
.filter_map(|(_, progress)| progress.page_index())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn peek_raw_enumeration_resume_state(&self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
self.selected_raw_enumeration_progress().map_or((None, None), |index| {
|
||||
let progress = &self.raw_enumeration_progress[index];
|
||||
(progress.cursor(), progress.page_index())
|
||||
})
|
||||
}
|
||||
|
||||
async fn scan_child_folder(
|
||||
@@ -1480,6 +1494,7 @@ impl FolderScanner {
|
||||
snapshot.info.last_update = Some(SystemTime::now());
|
||||
snapshot.info.snapshot_complete = false;
|
||||
let (cursor, page_index) = self.peek_raw_enumeration_resume_state();
|
||||
snapshot.info.scan_raw_enumeration_path_page_indexes = self.raw_enumeration_path_page_indexes();
|
||||
snapshot.info.scan_raw_enumeration_cursor = cursor;
|
||||
snapshot.info.scan_raw_enumeration_page_index = page_index;
|
||||
snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).ok()?;
|
||||
@@ -1976,6 +1991,9 @@ impl FolderScanner {
|
||||
let mut last_entry_progress = Instant::now();
|
||||
let mut raw_enumeration_complete = false;
|
||||
let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name);
|
||||
let raw_replay_floor = u64::try_from(raw_enumeration_committed_entries.len()).unwrap_or(u64::MAX);
|
||||
let mut raw_entries_observed = 0_u64;
|
||||
let mut raw_entries_charged = 0_u64;
|
||||
|
||||
let mut candidate_unclassified = mode != FolderScanMode::Namespace;
|
||||
let mut deferred_metadata = None;
|
||||
@@ -2061,8 +2079,13 @@ impl FolderScanner {
|
||||
continue;
|
||||
}
|
||||
if !replayed_metadata {
|
||||
let raw_entry_consumed_by_owner_index = raw_enumeration_committed_entries.contains(&file_name);
|
||||
if !raw_entry_consumed_by_owner_index {
|
||||
// A restored index appends one pending name per entry past
|
||||
// its old size, so charging only that tail keeps each
|
||||
// budget unit durable even if the listing order changed.
|
||||
raw_entries_observed = raw_entries_observed.saturating_add(1);
|
||||
let raw_entry_is_new =
|
||||
raw_entries_observed > raw_replay_floor && !raw_enumeration_committed_entries.contains(&file_name);
|
||||
if raw_entry_is_new {
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
@@ -2074,7 +2097,18 @@ impl FolderScanner {
|
||||
last_entry_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
let raw_entry_is_durable = self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
// Object metadata and the entries behind it are charged to
|
||||
// the object budget; this one covers listing work.
|
||||
let object_entry = file_name == STORAGE_FORMAT_FILE || deferred_metadata.is_some();
|
||||
if raw_entry_is_new && raw_entry_is_durable && !object_entry {
|
||||
raw_entries_charged = raw_entries_charged.saturating_add(1);
|
||||
let name_bytes = u64::try_from(file_name.len()).unwrap_or(u64::MAX);
|
||||
if !self.budget.try_charge_raw_entry(name_bytes, raw_entries_charged) {
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
}
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
}
|
||||
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
|
||||
@@ -2474,7 +2508,9 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
if raw_enumeration_complete {
|
||||
// A directory with queued children keeps its index until they end:
|
||||
// a restart replays this listing to reach the unfinished child.
|
||||
if raw_enumeration_complete && queued_folders.is_empty() {
|
||||
self.finish_raw_enumeration_parent(&folder.name);
|
||||
}
|
||||
|
||||
@@ -2683,10 +2719,12 @@ impl FolderScanner {
|
||||
"Scanner child folder scan failed"
|
||||
);
|
||||
self.coverage_gap |= forward_sweep;
|
||||
self.finish_raw_enumeration_parent(&folder_item.name);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
self.finish_raw_enumeration_parent(&folder_item.name);
|
||||
if outcome == FolderScanOutcome::Vanished {
|
||||
self.new_cache.delete_recursive(&h);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
@@ -3405,6 +3443,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
new_cache.info.scan_checkpoint = None;
|
||||
new_cache.info.scan_raw_enumeration_cursor = None;
|
||||
new_cache.info.scan_raw_enumeration_page_index = None;
|
||||
new_cache.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
if had_scan_checkpoint {
|
||||
global_metrics().record_scanner_checkpoint_cleared();
|
||||
@@ -3443,6 +3482,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
let root_hash = hash_path(&bucket_name);
|
||||
let root_has_progress = data_usage_root_has_progress(&root);
|
||||
let pending_heals_changed = scanner.pending_heals_changed;
|
||||
let raw_enumeration_path_page_indexes = scanner.raw_enumeration_path_page_indexes();
|
||||
let (raw_enumeration_cursor, raw_enumeration_page_index) = scanner.take_raw_enumeration_resume_state();
|
||||
let carry_forward_cache = ((raw_enumeration_cursor.is_some() || raw_enumeration_page_index.is_some())
|
||||
&& !root_has_progress)
|
||||
@@ -3465,6 +3505,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
}
|
||||
if raw_enumeration_page_index.is_some() {
|
||||
new_cache.info.scan_raw_enumeration_page_index = raw_enumeration_page_index;
|
||||
new_cache.info.scan_raw_enumeration_path_page_indexes = raw_enumeration_path_page_indexes;
|
||||
new_cache.info.scan_checkpoint = None;
|
||||
new_cache.info.scan_resume_after = None;
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
@@ -3495,6 +3536,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
global_metrics().record_scanner_checkpoint_cleared();
|
||||
}
|
||||
}
|
||||
new_cache.info.scan_resource_deferred = matches!(&e, ScannerError::ResourceDeferred);
|
||||
close_disk_guard.close().await;
|
||||
return Err(ScannerError::PartialCache(Box::new(std::mem::take(new_cache))));
|
||||
}
|
||||
@@ -3524,6 +3566,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
.insert(hash_path(&bucket_name).0, DataUsageEntry::default());
|
||||
}
|
||||
partial_cache.info.snapshot_complete = false;
|
||||
partial_cache.info.scan_resource_deferred = true;
|
||||
close_disk_guard.close().await;
|
||||
return Err(ScannerError::PartialCache(Box::new(partial_cache)));
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ use std::sync::Mutex;
|
||||
|
||||
pub(super) mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
mod fair_prefix_quantum;
|
||||
mod incremental_enumeration;
|
||||
mod listing_completion;
|
||||
mod metadata_visits;
|
||||
|
||||
@@ -51,6 +51,10 @@ impl Drop for ObservationGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn install_raw_entry_observer(root: PathBuf) -> ObservationGuard {
|
||||
install_raw_entry_budget(root, u64::MAX)
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn install_raw_entry_budget(root: PathBuf, limit: u64) -> ObservationGuard {
|
||||
*OBSERVATION.lock().expect("install raw-entry observation") = Some(Observation {
|
||||
root,
|
||||
@@ -83,6 +87,45 @@ pub(super) async fn read_bounded(path: &Path) -> Vec<u8> {
|
||||
bytes
|
||||
}
|
||||
|
||||
/// Creates a flat synthetic bucket and the checkpointable cache a first sweep
|
||||
/// starts from.
|
||||
pub(super) async fn seed_flat_bucket(disk_root: &Path, objects: usize) -> DataUsageCache {
|
||||
tokio::fs::create_dir(disk_root).await.expect("create fresh synthetic disk");
|
||||
for index in 0..objects {
|
||||
let object = format!("object-{index:04}");
|
||||
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
|
||||
let bytes = metadata_for_object_version("bucket", &object, Some(version));
|
||||
write_test_object_metadata_bytes(disk_root, "bucket", &object, &bytes).await;
|
||||
}
|
||||
let mut initial = DataUsageCache::default();
|
||||
initial.info.name = "bucket".to_string();
|
||||
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
|
||||
let plan = crate::data_usage_define::DataUsageScanPlanDigest([31; 32]);
|
||||
let identity = crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::from_u128(31),
|
||||
set_layout: crate::data_usage_define::DataUsageScanPlanDigest([32; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
assert_eq!(
|
||||
initial.prepare_bucket_checkpoint("bucket", 1, 0, source, plan, identity),
|
||||
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
initial.info.skip_healing = true;
|
||||
initial.replace("bucket", "", DataUsageEntry::default());
|
||||
initial
|
||||
}
|
||||
|
||||
pub(super) fn observed_raw_entries() -> u64 {
|
||||
OBSERVATION
|
||||
.lock()
|
||||
.expect("read raw-entry observation")
|
||||
.as_ref()
|
||||
.map_or(0, |observation| observation.entries)
|
||||
}
|
||||
|
||||
async fn round(request: &Request) -> serde_json::Value {
|
||||
assert!((1..=1024).contains(&request.objects));
|
||||
assert!((1..=4096).contains(&request.raw_entry_budget));
|
||||
@@ -90,31 +133,7 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
let disk_root = request.workspace.join("disk");
|
||||
let cache_path = request.workspace.join("cache.bin");
|
||||
if request.round == 0 {
|
||||
tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk");
|
||||
for index in 0..request.objects {
|
||||
let object = format!("object-{index:04}");
|
||||
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
|
||||
let bytes = metadata_for_object_version("bucket", &object, Some(version));
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await;
|
||||
}
|
||||
let mut initial = DataUsageCache::default();
|
||||
initial.info.name = "bucket".to_string();
|
||||
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
|
||||
let plan = crate::data_usage_define::DataUsageScanPlanDigest([31; 32]);
|
||||
let identity = crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::from_u128(31),
|
||||
set_layout: crate::data_usage_define::DataUsageScanPlanDigest([32; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
assert_eq!(
|
||||
initial.prepare_bucket_checkpoint("bucket", 1, 0, source, plan, identity),
|
||||
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
initial.info.skip_healing = true;
|
||||
initial.replace("bucket", "", DataUsageEntry::default());
|
||||
let initial = seed_flat_bucket(&disk_root, request.objects).await;
|
||||
tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
|
||||
.await
|
||||
.expect("persist initial cache");
|
||||
@@ -137,10 +156,11 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
&parent,
|
||||
crate::scanner_budget::ScannerCycleBudgetConfig {
|
||||
max_objects: Some(request.raw_entry_budget),
|
||||
max_entries: Some(request.raw_entry_budget),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let _observation_guard = install_raw_entry_budget(disk.path(), request.raw_entry_budget);
|
||||
let _observation_guard = install_raw_entry_observer(disk.path());
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user