mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-03 01:23:53 +00:00
Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f2e8cd69d2 | |||
| 2f55cf69e1 | |||
| fff6e94378 | |||
| 56403813cc | |||
| 41b13aeb8b | |||
| 88587a8f94 | |||
| 8c8b5da63f | |||
| 00a238f838 | |||
| 965d805a4f | |||
| 9d66e04ae3 | |||
| 12c58d25b5 | |||
| 1ac1d883f4 | |||
| 45ba33291a | |||
| 636eded277 | |||
| ed4099ebe1 | |||
| 3a1fac16e3 | |||
| b00e85f1a4 | |||
| ecbddc188a | |||
| 2d63931bba | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 | |||
| 3c0829049d | |||
| 890a463218 | |||
| 25f45f6309 | |||
| e3700614a1 | |||
| d4ab7704b3 | |||
| 422e724051 | |||
| e0bf54197d | |||
| b4af43c884 | |||
| 1cb4c0ebe2 | |||
| e5563e6004 | |||
| 0a72b29a34 | |||
| 51275cb7d8 | |||
| 3bf270b49f | |||
| d8d064bd18 | |||
| e5653ca055 | |||
| 6413a11889 | |||
| 9e4ea9fdd1 | |||
| 1a841fb7c7 | |||
| b8ee0301e9 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
|
||||
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
|
||||
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -976,6 +976,12 @@ test-group = 'e2e-vault'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
# scenarios exercise write/read, anonymous denial and retained bytes on restart.
|
||||
[profile.core-contract]
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,8 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -296,6 +296,32 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
@@ -743,12 +769,10 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -823,12 +847,11 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
+311
-5
@@ -814,13 +814,18 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -869,6 +874,221 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1028,6 +1248,10 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1068,6 +1292,45 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1084,16 +1347,59 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status)
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
+95
-34
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
@@ -193,13 +193,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -233,11 +233,11 @@ jobs:
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ test-and-lint, classify-changes ]
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
@@ -293,14 +293,14 @@ jobs:
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
@@ -321,13 +321,13 @@ jobs:
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
@@ -368,7 +368,7 @@ jobs:
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/test-and-lint/core-test-listing.json \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
@@ -449,7 +449,7 @@ jobs:
|
||||
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -497,13 +497,72 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -540,12 +599,12 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
@@ -555,14 +614,14 @@ jobs:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -631,7 +690,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -783,7 +842,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -978,7 +1037,7 @@ jobs:
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
@@ -1389,12 +1448,13 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1422,10 +1482,11 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
|
||||
@@ -329,13 +329,75 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: build-check
|
||||
needs: [ build-check, release-certification ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -443,6 +505,14 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -467,7 +537,9 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -493,6 +565,114 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,32 +33,6 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,30 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,7 +33,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -30,9 +30,102 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
@@ -346,12 +346,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,6 +360,90 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
+210
-14
@@ -225,10 +225,57 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: resolve
|
||||
needs: [ resolve, release-certification ]
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -494,6 +541,55 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -531,10 +627,14 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -549,18 +649,6 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -614,6 +702,114 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
Generated
+386
-179
File diff suppressed because it is too large
Load Diff
+14
-18
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -175,9 +175,9 @@ tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -217,7 +217,7 @@ chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -253,9 +253,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -319,7 +319,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -334,10 +334,10 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.0"
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
test-case = "3.4.1"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -356,7 +356,7 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.3"
|
||||
zip = "9.2.2"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -384,9 +384,9 @@ russh-sftp = "3.0.1"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -427,7 +427,3 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -16,8 +16,193 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -12,11 +12,249 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -1636,20 +1636,53 @@ async fn hot_window_with_old_followers(
|
||||
let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
|
||||
let scheduler_config = run.assert_default_cadence().await?;
|
||||
let baseline = run.quiet_baseline().await?;
|
||||
let leader = run.leader().await?;
|
||||
let leader_pid = run.pid(leader)?;
|
||||
// A leader-lock refresh gap during old-binary replacement can make
|
||||
// `leader()` briefly observe a different node. Poll until the coordinator
|
||||
// is stable before comparing against the measured fleet. A persistent
|
||||
// change to an old-binary node is a genuine election outcome and fails.
|
||||
let (leader, leader_pid) = if mixed {
|
||||
let expected_leader = usize::try_from(
|
||||
mixed_fleet["leader"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader")?,
|
||||
)?;
|
||||
let expected_pid = mixed_fleet["leader_pid"]
|
||||
.as_u64()
|
||||
.ok_or("mixed fleet omitted its measured leader pid")?;
|
||||
let mut resolved = None;
|
||||
for _ in 0..20 {
|
||||
let candidate = run.leader().await?;
|
||||
let candidate_pid = u64::from(run.pid(candidate)?);
|
||||
if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
|
||||
resolved = Some((candidate, run.pid(candidate)?));
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
match resolved {
|
||||
Some(pair) => pair,
|
||||
None => {
|
||||
let current = match run.leader().await {
|
||||
Ok(node) => Some((node, run.pid(node).ok())),
|
||||
Err(_) => None,
|
||||
};
|
||||
return Err(format!(
|
||||
"the hot window did not retain its measured new coordinator after stabilization \
|
||||
(expected node {expected_leader} pid {expected_pid}, observed {current:?})"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let leader = run.leader().await?;
|
||||
(leader, run.pid(leader)?)
|
||||
};
|
||||
let producer = (leader + 1) % 4;
|
||||
let producers = if fault.is_some() {
|
||||
vec![producer, (leader + 2) % 4]
|
||||
} else {
|
||||
vec![producer]
|
||||
};
|
||||
if mixed
|
||||
&& (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
|
||||
|| mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
|
||||
{
|
||||
return Err("the hot window did not retain its measured new coordinator".into());
|
||||
}
|
||||
let producer_pid = run.pid(producer)?;
|
||||
let producer_address = run.peer_address(producer);
|
||||
let [cold_a, cold_b, hot] = run.buckets.clone();
|
||||
|
||||
@@ -2146,3 +2146,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -1112,8 +1112,9 @@ async fn wait_for_upgrade_writes(
|
||||
match response {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|
||||
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
|
||||
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|
||||
{
|
||||
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -1901,6 +1902,15 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
|
||||
}
|
||||
@@ -5200,6 +5210,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
|
||||
@@ -747,6 +747,16 @@ async fn list_path_raw_inner(
|
||||
// Start every missing head read in the same round so one stalled
|
||||
// disk cannot multiply the wait budget by the erasure-set width.
|
||||
// Outcomes are still consumed below in stable disk-index order.
|
||||
if !revjob_rx.is_cancelled() {
|
||||
for (i, reader) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_none()
|
||||
&& pending_entries[i].is_none()
|
||||
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
|
||||
{
|
||||
pending_entries[i] = Some(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
|
||||
if errs[i].is_some() || pending_entries[i].is_some() {
|
||||
return None;
|
||||
@@ -878,7 +888,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
if entry.name == current.name {
|
||||
// If exact match, we agree.
|
||||
if current.matches_strict(&entry) {
|
||||
if current.matches_strict_borrowed(&entry) {
|
||||
top_entries[i] = Some(entry);
|
||||
agree += 1;
|
||||
} else {
|
||||
@@ -986,7 +996,9 @@ async fn list_path_raw_inner(
|
||||
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
let _ = r.skip(1).await;
|
||||
if !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
@@ -1009,7 +1021,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if top_entries[i].is_some() {
|
||||
if top_entries[i].is_some() && !r.skip_current() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1637,6 +1637,7 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1708,6 +1709,7 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1778,6 +1780,7 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2009,6 +2012,7 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2023,6 +2027,7 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
|
||||
@@ -445,6 +445,10 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
});
|
||||
// A successful lock RPC over a cached or fresh channel proves the peer is
|
||||
// reachable; reconcile the internode peer-health gauge so it does not stay
|
||||
// stale when no fresh dial fires (issue #9119).
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
@@ -672,6 +676,7 @@ impl RemoteClient {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
late(response).await;
|
||||
|
||||
@@ -3506,6 +3506,7 @@ mod tests {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -6827,11 +6838,48 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder)
|
||||
.map_err(|error| Error::other(format!("pool metadata legacy source is invalid: {error}")))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7222,6 +7270,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7276,6 +7326,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7289,6 +7341,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7302,19 +7358,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, _, mut selected) =
|
||||
let (_, selected_raw, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -14534,6 +14607,23 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14544,10 +14634,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != *canonical {
|
||||
if confirmed != repair_data {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -28353,7 +28443,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28387,6 +28477,88 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28429,7 +28601,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28838,17 +29010,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28873,6 +29045,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
@@ -29351,6 +29560,53 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
|
||||
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -853,6 +853,8 @@ pub(crate) struct DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
|
||||
/// Caller API operations currently executing through the disk health wrapper.
|
||||
api_waiting: AtomicU32,
|
||||
/// Operations rejected because the disk was unavailable for the caller.
|
||||
@@ -880,6 +882,8 @@ impl Default for DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex::new(None),
|
||||
api_waiting: AtomicU32::new(0),
|
||||
total_errors_availability: AtomicU64::new(0),
|
||||
total_errors_timeout: AtomicU64::new(0),
|
||||
@@ -1573,6 +1577,20 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScopedDiskOperationTimeout {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
previous: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedDiskOperationTimeout {
|
||||
fn drop(&mut self) {
|
||||
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1914,6 +1932,23 @@ impl LocalDiskWrapper {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
|
||||
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
|
||||
// The metrics epoch follows this drive across clones and reconnects.
|
||||
// Its scoped override cannot shorten unrelated healthy drives' I/O.
|
||||
let previous = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.replace(timeout);
|
||||
ScopedDiskOperationTimeout {
|
||||
metrics: Arc::clone(&self.metrics),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -2398,6 +2433,10 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2634,6 +2673,13 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#[cfg(test)]
|
||||
let timeout_duration = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
@@ -6040,6 +6086,67 @@ mod tests {
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
|
||||
let directories = [
|
||||
tempfile::tempdir().expect("faulted drive"),
|
||||
tempfile::tempdir().expect("healthy drive"),
|
||||
];
|
||||
let mut wrappers = Vec::new();
|
||||
for directory in &directories {
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
|
||||
wrappers.push(LocalDiskWrapper::new(disk, false));
|
||||
}
|
||||
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::clone(&wrappers[0].disk),
|
||||
false,
|
||||
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
|
||||
);
|
||||
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
|
||||
tokio::time::pause();
|
||||
let operation = || async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(())
|
||||
};
|
||||
let (faulted, healthy) = tokio::join!(
|
||||
reconnected.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
wrappers[1].track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
);
|
||||
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
|
||||
healthy.expect("healthy drive keeps its caller's deadline");
|
||||
let shorter_started = tokio::time::Instant::now();
|
||||
let shorter = wrappers[0]
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_millis(500),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
|
||||
assert!(shorter_started.elapsed() < Duration::from_secs(1));
|
||||
wrappers[0]
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
|
||||
.await
|
||||
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
|
||||
drop(scope);
|
||||
reconnected
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
|
||||
.await
|
||||
.expect("dropping the scope restores reconnected callers' deadlines");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ignored_timeout_does_not_mark_drive_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume predicate"
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
|
||||
+1080
-236
File diff suppressed because it is too large
Load Diff
@@ -175,17 +175,30 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
old_metadata: &[u8],
|
||||
sync: bool,
|
||||
) -> std::io::Result<PathBuf> {
|
||||
let Some(staging_parent) = staging_file_path.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
|
||||
};
|
||||
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
|
||||
&& let Err(err) = std::fs::write(&backup_path, old_metadata)
|
||||
{
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = write_res {
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -356,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(RenameDataResp),
|
||||
Committed(Box<RenameDataResp>),
|
||||
CommitErr(DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
#[cfg(test)]
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
@@ -540,7 +556,9 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
|
||||
Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
|
||||
Some(Arc::new(
|
||||
pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -741,11 +759,6 @@ impl LocalDisk {
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -785,10 +798,11 @@ impl LocalDisk {
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
)?;
|
||||
// An existing xl.meta can share versions or require rollback
|
||||
// restoration when a post-rename directory sync fails.
|
||||
if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
// An existing xl.meta that replaces an existing version slot
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -811,7 +825,19 @@ impl LocalDisk {
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
if no_inline
|
||||
&& sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
@@ -830,13 +856,31 @@ impl LocalDisk {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& (sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
@@ -846,49 +890,157 @@ impl LocalDisk {
|
||||
if let Err(err) = sync_result
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
|
||||
}
|
||||
|
||||
let guard = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
guard,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
std::fs::remove_file(&src_file_path_owned)?;
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if has_data_dir_path_owned.is_some()
|
||||
&& let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}))
|
||||
{
|
||||
return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) =
|
||||
fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
|
||||
{
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
return Err(err);
|
||||
let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
destination_was_absent,
|
||||
staged_rollback_path.as_ref(),
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline {
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path_owned.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
@@ -897,7 +1049,7 @@ impl LocalDisk {
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))
|
||||
})))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
@@ -917,7 +1069,23 @@ impl LocalDisk {
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(resp);
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -925,11 +1093,269 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
|
||||
std::panic::resume_unwind(panic_payload);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = restored_fi;
|
||||
fi = *restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else if let Some(publication) = multipart_publication.as_ref()
|
||||
&& no_inline
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
AbortBeforeDataRename(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
RestoreTargetAfterError(DiskError, bool),
|
||||
}
|
||||
let publication_owned = Arc::clone(publication);
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let outcome = os::run_blocking_namespace_operation_with_sync(
|
||||
mutation_lease.clone(),
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let skip_parent_owned = if has_dst_buf_owned.is_some() {
|
||||
dst_parent_owned.clone()
|
||||
} else {
|
||||
dst_volume_dir_owned.clone()
|
||||
};
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf_owned.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
if let Err(err) = xlmeta.add_version(fi_moved) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
|
||||
}
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf_owned = match xlmeta.marshal_msg() {
|
||||
Ok(buf) => buf,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
|
||||
};
|
||||
|
||||
let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
|
||||
&new_dst_buf_owned,
|
||||
has_dst_buf_owned.as_deref(),
|
||||
&dst_file_path_owned,
|
||||
&directory_guard_owned,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
// When presynced, shard payloads were synced during UploadPart and
|
||||
// prepare_and_install_head_on_thread just fsynced src_data_path after
|
||||
// writing RETIREMENT_ANCHOR.
|
||||
if sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
|
||||
&& let Err(err) = os::sync_dir_files_std(src_data_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
let directory_guard_owned = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
directory_guard_owned,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let Some(src_parent) = src_data_path.parent() else {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
|
||||
};
|
||||
let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
|
||||
directory_guard_owned,
|
||||
DiskError::Unexpected,
|
||||
));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent_owned,
|
||||
&publication_root_owned,
|
||||
&directory_guard_owned,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
|
||||
}
|
||||
if has_dst_buf_owned.is_none() {
|
||||
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir_owned) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir_limited_std(dir) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
|
||||
to_file_error(err).into(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if dir == dst_volume_dir_owned.as_path() {
|
||||
break;
|
||||
}
|
||||
ancestor = dir.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))),
|
||||
Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
match outcome {
|
||||
MultipartFusedCommitOutcome::Committed(resp) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
|
||||
MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
|
||||
if !invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
|
||||
if invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
@@ -1053,71 +1479,81 @@ impl LocalDisk {
|
||||
// after the commit rename, and a crash before the rename means the
|
||||
// PUT was never acknowledged. A metadata commit: relaxed tiers
|
||||
// leave it to the page cache.
|
||||
if let Some(publication) = multipart_publication.as_ref() {
|
||||
pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
|
||||
}
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
// The tmp xl.meta write and the shard-file fdatasync are independent
|
||||
// (disjoint paths) and both only need to be durable before the commit
|
||||
// renames below, so run them concurrently to drop a blocking
|
||||
// round-trip from the PUT commit critical path (rustfs/backlog#922
|
||||
// step 2). The "contents durable -> rename -> dst dir fsync" ordering
|
||||
// is unchanged — both futures complete before any rename — which the
|
||||
// rename_data crash-consistency harness (backlog#935) exercises.
|
||||
//
|
||||
// Shard durability: once rename_data succeeds the write is
|
||||
// acknowledged, so data must not live only in the page cache.
|
||||
// Multipart parts were already synced during rename_part, so their
|
||||
// fdatasync here is a cheap no-op. A missing source dir is left for the
|
||||
// rename below to report through the existing rollback path. Payload
|
||||
// durability is kept by both strict and relaxed.
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = multipart_publication
|
||||
.as_ref()
|
||||
.map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(all(not(test), target_os = "linux"))]
|
||||
let probe_active = false;
|
||||
let shard_sync = async {
|
||||
if durability.syncs_data_shards()
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
#[cfg(target_os = "linux")]
|
||||
if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
if multipart_publication.is_none()
|
||||
&& let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
// Surface a tmp-meta failure first (its prior serial position), then a
|
||||
// shard-sync failure; either aborts before any rename, exactly as the
|
||||
// sequential version did.
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
let prepared_metadata_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
|
||||
None => prepared_metadata_source,
|
||||
let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
|
||||
let dst_file_path_ref = &dst_file_path;
|
||||
let dir_guard_ref = &commit.directory_guard;
|
||||
let lease_clone = commit.mutation_lease.clone();
|
||||
let prepare_fut = pin_multipart_future(|| {
|
||||
publication.prepare_and_install_head(
|
||||
&new_dst_buf,
|
||||
has_dst_buf.as_deref(),
|
||||
dst_file_path_ref,
|
||||
dir_guard_ref,
|
||||
lease_clone,
|
||||
)
|
||||
});
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
} else {
|
||||
let tmp_meta_sync = if durability.syncs_commit_metadata() {
|
||||
SyncMode::FileOnly
|
||||
} else {
|
||||
SyncMode::None
|
||||
};
|
||||
let tmp_meta_write = {
|
||||
let src_file_path = commit.source_metadata.to_path_buf();
|
||||
let dst_file_path = dst_file_path.clone();
|
||||
let owned_directory_guard = commit.directory_guard.clone();
|
||||
let owned_mutation_lease = commit.mutation_lease.clone();
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&owned_directory_guard,
|
||||
)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
|
||||
Ok(prepared_metadata_source)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
};
|
||||
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
|
||||
let prepared_metadata_source = tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
prepared_metadata_source
|
||||
};
|
||||
commit.directory_guard = remove_dst_base_before_commit(
|
||||
dst_path,
|
||||
@@ -1457,13 +1893,22 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
|
||||
return Err(error);
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
Ok(RenameDataResp {
|
||||
@@ -1663,13 +2108,8 @@ impl LocalDisk {
|
||||
#[cfg(not(windows))]
|
||||
let backup_path = {
|
||||
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata)?
|
||||
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
if sync {
|
||||
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
}
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
|
||||
@@ -1000,6 +1000,25 @@ pub(super) fn inspection_with_markers(
|
||||
Ok((inspection, has_physical_data_dirs_or_markers))
|
||||
}
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Native MPU source closure has its own conditional protocol. Generic
|
||||
// object recovery must never restore its head or acquire SourceRW under
|
||||
@@ -1523,6 +1542,9 @@ impl LocalDisk {
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
}
|
||||
|
||||
let (mut observed, mut has_data_dir_work) = self
|
||||
.delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
|
||||
@@ -1911,8 +1933,14 @@ impl LocalDisk {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let mut file_leases = Vec::with_capacity(3);
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let mut file_leases = Vec::with_capacity(5);
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
let path = rollback.join(name);
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
|
||||
file_leases.push((path, lease));
|
||||
@@ -2132,119 +2160,151 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let (old_dir, journal_raw) = match (|| -> Result<Option<(Option<Uuid>, Vec<u8>)>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let deleted_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| v.header.version_id.unwrap_or_default())
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
if !meta.versions.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut predecessors = BTreeSet::new();
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let Some(id) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| Uuid::parse_str(name).ok())
|
||||
.filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let prepared = regular_bytes(&entry.path().join(PREPARED))?;
|
||||
let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let surviving: HashSet<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
|
||||
.collect();
|
||||
deleted_headers.retain(|header| !surviving.contains(header));
|
||||
let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
|
||||
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let file_name = entry.file_name();
|
||||
let Some(name_str) = file_name.to_str() else {
|
||||
sole_transaction = false;
|
||||
continue;
|
||||
};
|
||||
let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let dir_path = entry.path();
|
||||
let prepared = regular_bytes(&dir_path.join(PREPARED))?;
|
||||
let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
if sole_transaction && name_str == id.hyphenated().to_string() && old_dir != Some(id) {
|
||||
let is_surviving_data_dir = original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& prepared.is_none()
|
||||
&& receipt.is_none()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
}
|
||||
}
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: None,
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
sole_transaction &= predecessors.is_empty();
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: target.as_deref().map(digest),
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
let raw = journal_bytes(&journal)?;
|
||||
Ok(Some((old_dir, target, raw, journal, sole_transaction)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
Ok(Some((old_dir, journal_bytes(&journal)?)))
|
||||
})() {
|
||||
Ok(Some(pair)) => pair,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = tempfile::Builder::new()
|
||||
.prefix("delete-preparation-")
|
||||
.tempdir_in(&staging_root)?;
|
||||
for (name, bytes) in [
|
||||
(STORAGE_FORMAT_FILE_BACKUP, buf.as_slice()),
|
||||
(PREPARED, journal_raw.as_slice()),
|
||||
] {
|
||||
let mut file = std::fs::File::create(staged.path().join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
|
||||
let staged_dir = staged.path().to_path_buf();
|
||||
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, &buf)?;
|
||||
if sync && !buf.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
}
|
||||
let mut written_files = Vec::with_capacity(2);
|
||||
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
|
||||
let Some(bytes) = bytes else {
|
||||
continue;
|
||||
};
|
||||
let mut file = std::fs::File::create(staged_dir.join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
written_files.push(file);
|
||||
}
|
||||
}
|
||||
for file in written_files {
|
||||
file.sync_data()?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(staged.path())?;
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
}
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
@@ -2257,27 +2317,62 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
let staged_dir = staged.keep();
|
||||
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
|
||||
let _ = std::fs::remove_dir_all(&staged_dir);
|
||||
return Err(err);
|
||||
}
|
||||
std::fs::rename(staged.path(), &destination)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&file_path)?;
|
||||
let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(uuid) = old_dir {
|
||||
let old_path = file_path.join(uuid.to_string());
|
||||
if old_path.is_dir() {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
if let Some(old_path) = old_data_path {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let reserve_res = (|| -> std::io::Result<()> {
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = reserve_res {
|
||||
let _ = std::fs::remove_file(&reserved_marker);
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
if let Some(ref target_bytes) = target {
|
||||
let target_in_rollback = destination.join(TARGET);
|
||||
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
|
||||
let rename_res = std::fs::rename(&linked_xl, &xl_path);
|
||||
if rename_res.is_err() {
|
||||
let _ = std::fs::remove_file(&linked_xl);
|
||||
}
|
||||
rename_res
|
||||
} else {
|
||||
(|| -> std::io::Result<()> {
|
||||
let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
|
||||
if sync && !target_bytes.is_empty() {
|
||||
staged_xl.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_xl.path(), &xl_path)
|
||||
})()
|
||||
};
|
||||
if let Err(err) = write_res {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
} else if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
@@ -2298,6 +2393,7 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
put_prepared_delete_receipt(destination, (journal, sole_transaction));
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
@@ -2343,6 +2439,26 @@ impl LocalDisk {
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
|
||||
&& rollback.join(PREPARED).is_file()
|
||||
{
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: false,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged.path(), &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
return Ok(Ok(Some((journal, sole_transaction, true))));
|
||||
}
|
||||
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
|
||||
Ok(Some(bytes)) => bytes,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
@@ -2360,12 +2476,18 @@ impl LocalDisk {
|
||||
{
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let current_exists = match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
let target_satisfied = match journal.target_digest {
|
||||
None => match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => !exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
},
|
||||
Some(expected_target) => match regular_bytes(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(Some(bytes)) => digest(&bytes) == expected_target,
|
||||
Ok(None) => false,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
},
|
||||
};
|
||||
let unversioned_target_absent = journal.target_digest.is_none() && !current_exists;
|
||||
if !unversioned_target_absent {
|
||||
if !target_satisfied {
|
||||
return Ok(Ok(None));
|
||||
}
|
||||
let mut sole_transaction = journal.predecessors.is_empty();
|
||||
@@ -2382,8 +2504,15 @@ impl LocalDisk {
|
||||
&& id != transaction
|
||||
&& !journal.data_dirs.contains(&id)
|
||||
{
|
||||
sole_transaction = false;
|
||||
break;
|
||||
let dir_path = entry.path();
|
||||
let is_surviving_data_dir = journal.original_versions.iter().any(|v| v.data_dir == Some(id))
|
||||
&& !dir_path.join(PREPARED).exists()
|
||||
&& !dir_path.join(DELETE_QUORUM_COMMITTED_FILE).exists()
|
||||
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
|
||||
if !is_surviving_data_dir {
|
||||
sole_transaction = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2406,7 +2535,7 @@ impl LocalDisk {
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
Ok(Ok(Some((journal, sole_transaction, unversioned_target_absent))))
|
||||
Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2458,12 +2587,15 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let has_target = journal.target_digest.is_some();
|
||||
let journal_for_receipt = journal.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
@@ -2474,11 +2606,13 @@ impl LocalDisk {
|
||||
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let mut had_physical_data_dirs = false;
|
||||
for data_dir in &data_dirs {
|
||||
for base in [&root, &rollback] {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in entries {
|
||||
@@ -2519,23 +2653,45 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !data_dirs.is_empty() && sync && root.is_dir() {
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
}
|
||||
if has_target || had_physical_data_dirs {
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal_for_receipt,
|
||||
finished: true,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged_receipt.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged_receipt.path(), &receipt_path)?;
|
||||
let _ = std::fs::File::create(rollback.join(FINISHED))?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
}
|
||||
for name in [TARGET, PREPARED, FINISHED] {
|
||||
let path = rollback.join(name);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
|
||||
ignore_not_found(std::fs::remove_file(&receipt_path))?;
|
||||
for dir_path in &prune_paths {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
|
||||
if let Err(err) = std::fs::remove_dir(dir_path) {
|
||||
if is_dir_not_empty_error(&err) {
|
||||
break;
|
||||
@@ -6282,6 +6438,214 @@ mod tests {
|
||||
b"outside bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_marker_and_publish_fast_path() {
|
||||
let (_temp, disk, info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object root");
|
||||
let transaction = Uuid::new_v4();
|
||||
let mut marker = FileInfo {
|
||||
name: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: info.mod_time.map(|t| t + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &marker, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused delete marker preparation must succeed");
|
||||
assert_eq!(fused, Some(()), "versioned delete marker must use fused path");
|
||||
assert!(root.join(transaction.to_string()).join(TARGET).is_file());
|
||||
assert!(root.join(STORAGE_FORMAT_FILE).is_file());
|
||||
|
||||
let published = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("versioned target must satisfy local publish");
|
||||
assert!(published.1, "sole transaction must be true");
|
||||
assert!(published.2, "target_satisfied must be true");
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &published.0, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
let final_meta = FileMeta::load(&fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("read final xl.meta"))
|
||||
.expect("decode final xl.meta");
|
||||
assert_eq!(final_meta.versions.len(), 2);
|
||||
assert_eq!(final_meta.versions[0].header.version_id, marker.version_id);
|
||||
marker.deleted = false;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
|
||||
let (_temp, disk, mut v1, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
// Configure v1 and v2 with distinct physical data directories so deleting v2
|
||||
// exercises both `old_dir` reservation and `is_surviving_data_dir`.
|
||||
v1.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut v1.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let v1_dir = Uuid::new_v4();
|
||||
v1.data_dir = Some(v1_dir);
|
||||
fs::create_dir_all(root.join(v1_dir.to_string()))
|
||||
.await
|
||||
.expect("create v1 dir");
|
||||
fs::write(root.join(v1_dir.to_string()).join("part.1"), b"v1-data")
|
||||
.await
|
||||
.expect("write v1 part");
|
||||
|
||||
let mut v2 = v1.clone();
|
||||
v2.version_id = Some(Uuid::new_v4());
|
||||
let v2_dir = Uuid::new_v4();
|
||||
v2.data_dir = Some(v2_dir);
|
||||
fs::create_dir_all(root.join(v2_dir.to_string()))
|
||||
.await
|
||||
.expect("create v2 dir");
|
||||
fs::write(root.join(v2_dir.to_string()).join("part.1"), b"v2-data")
|
||||
.await
|
||||
.expect("write v2 part");
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(v1.clone()).expect("add v1");
|
||||
meta.add_version(v2.clone()).expect("add v2");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused versioned delete preparation must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
assert!(
|
||||
root.join(v2_dir.to_string())
|
||||
.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"))
|
||||
.is_file(),
|
||||
"deleted version data_dir must have recovery marker"
|
||||
);
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("versioned target must satisfy local publish");
|
||||
assert!(sole_transaction, "surviving v1 data_dir must not clear sole_transaction");
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_some());
|
||||
|
||||
// Simulate interruption during finish-before-finalize right after TARGET is removed
|
||||
// (after finished receipt + FINISHED marker were persisted, before receipt_path is removed).
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let finished_receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: true,
|
||||
};
|
||||
fs::write(
|
||||
rollback.join(DELETE_QUORUM_COMMITTED_FILE),
|
||||
serde_json::to_vec(&finished_receipt).expect("serialize finished receipt"),
|
||||
)
|
||||
.await
|
||||
.expect("write finished receipt");
|
||||
fs::write(rollback.join(FINISHED), []).await.expect("write FINISHED");
|
||||
fs::remove_file(rollback.join(STORAGE_FORMAT_FILE_BACKUP))
|
||||
.await
|
||||
.expect("remove backup");
|
||||
fs::remove_file(rollback.join(TARGET)).await.expect("remove TARGET");
|
||||
|
||||
let inspected = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("interrupted post-TARGET state must remain inspectable without FileCorrupt");
|
||||
assert_eq!(inspected.state, DeleteRecoveryState::CommitFinished);
|
||||
|
||||
// Complete cleanup via try_finish_and_finalize_prepared_delete_local.
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed after interruption");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(v2_dir.to_string()).exists(), "deleted v2 data_dir must be removed");
|
||||
assert!(
|
||||
root.join(v1_dir.to_string()).join("part.1").is_file(),
|
||||
"surviving v1 data_dir must remain intact"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
|
||||
let (_temp, disk, mut info, _) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
|
||||
info.data = None;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut info.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
|
||||
);
|
||||
let data_dir = Uuid::new_v4();
|
||||
info.data_dir = Some(data_dir);
|
||||
fs::create_dir_all(root.join(data_dir.to_string()))
|
||||
.await
|
||||
.expect("create data_dir");
|
||||
fs::write(root.join(data_dir.to_string()).join("part.1"), b"physical-payload")
|
||||
.await
|
||||
.expect("write part.1");
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(info.clone()).expect("add version");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
|
||||
.await
|
||||
.expect("write xl.meta");
|
||||
|
||||
let transaction = Uuid::new_v4();
|
||||
let opts = DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let fused = disk
|
||||
.try_fused_recoverable_delete_version("recovery", "object", &info, &opts, transaction, None)
|
||||
.await
|
||||
.expect("fused unversioned physical delete must succeed");
|
||||
assert_eq!(fused, Some(()));
|
||||
|
||||
let (journal, sole_transaction, target_satisfied) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("local publish must succeed")
|
||||
.expect("absent target must satisfy local publish");
|
||||
assert!(sole_transaction);
|
||||
assert!(target_satisfied);
|
||||
assert!(journal.target_digest.is_none());
|
||||
|
||||
let rollback = root.join(transaction.to_string());
|
||||
assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
|
||||
|
||||
let finalized = disk
|
||||
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("local finalize must succeed");
|
||||
assert!(finalized);
|
||||
assert!(!rollback.exists(), "rollback directory must be pruned");
|
||||
assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
|
||||
assert!(!root.exists(), "empty object directory must be pruned");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -421,6 +421,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
|
||||
super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
|
||||
let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
let source = store.source.clone();
|
||||
let base = store.base.clone();
|
||||
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
root: os::PublicationRoot,
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
|
||||
@@ -238,9 +238,192 @@ pub(super) fn read_optional_in_pinned(
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct FileStamp {
|
||||
len: u64,
|
||||
modified: Option<std::time::SystemTime>,
|
||||
#[cfg(unix)]
|
||||
ino: u64,
|
||||
}
|
||||
|
||||
impl FileStamp {
|
||||
fn from_path(path: &Path) -> Option<Self> {
|
||||
let meta = std::fs::symlink_metadata(path).ok()?;
|
||||
Some(Self {
|
||||
len: meta.len(),
|
||||
modified: meta.modified().ok(),
|
||||
#[cfg(unix)]
|
||||
ino: {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
meta.ino()
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveMultipartSourceEntry {
|
||||
raw: Bytes,
|
||||
fi: FileInfo,
|
||||
head_stamp: FileStamp,
|
||||
unsettled_parts: std::collections::HashSet<usize>,
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
fi,
|
||||
head_stamp,
|
||||
unsettled_parts: std::collections::HashSet::new(),
|
||||
parts: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
source: &Path,
|
||||
part_number: Option<usize>,
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
map.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
|
||||
return None;
|
||||
}
|
||||
Some(part)
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
if !initial && has_active_multipart_source_head(source) {
|
||||
return Ok(());
|
||||
}
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
@@ -440,6 +623,7 @@ impl SourceStore {
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
@@ -470,7 +654,7 @@ impl SourceStore {
|
||||
let terminal = terminal_record_path(&source, state);
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
|
||||
Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
@@ -539,7 +723,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -551,6 +735,7 @@ impl SourceStore {
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
@@ -582,9 +767,10 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
@@ -1097,6 +1283,82 @@ pub(super) fn load_source(
|
||||
})
|
||||
}
|
||||
|
||||
fn load_uninitialized_source(
|
||||
raw: Bytes,
|
||||
path: &str,
|
||||
disk_id: Uuid,
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let sha = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let current = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
native_source_protocol(¤t)?;
|
||||
if source_status(¤t)?.is_some()
|
||||
|| publication_identity(¤t)?.is_some()
|
||||
|| SourceConditionV1::take(&mut current.clone())?.is_some()
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-bucket")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let object = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-object")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let condition = capture_source_condition(bucket, object, path, ¤t, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
|
||||
if condition.generation.epoch != 0 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let slot = condition
|
||||
.lineage
|
||||
.geometry
|
||||
.distribution
|
||||
.iter()
|
||||
.position(|index| *index == current.erasure.index)
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
|
||||
let snapshot = Some(MultipartSourceSnapshot {
|
||||
disk_id,
|
||||
physical_slot: slot,
|
||||
lineage: condition.lineage,
|
||||
generation: condition.generation,
|
||||
revision: 0,
|
||||
phase: MultipartSourcePhase::Active,
|
||||
state_sha256: [0; 32],
|
||||
head_sha256: Some(sha),
|
||||
inventory_sha256: inventory,
|
||||
capability_sha256: None,
|
||||
publication: None,
|
||||
});
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state: None,
|
||||
head: Some(SourceHead {
|
||||
sha256: sha,
|
||||
length,
|
||||
raw: Some(raw.clone()),
|
||||
}),
|
||||
original: Some(raw),
|
||||
info: Some(current),
|
||||
snapshot,
|
||||
head_matches_state: true,
|
||||
head_matches_activation: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -1323,41 +1585,6 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
/// Only native source validation may create provenance. Ordinary TMP/heal
|
||||
/// publication can retain a valid old local anchor, never issue/rebind one.
|
||||
pub async fn write_retirement_anchor(
|
||||
&self,
|
||||
state: &SourceState,
|
||||
publication: &PublicationIdentity,
|
||||
directory: &Path,
|
||||
lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<()> {
|
||||
if state.lineage != publication.lineage
|
||||
|| state.generation != publication.source_generation
|
||||
|| state.disk_id != self.disk_id
|
||||
|| state.physical_slot != self.physical_slot
|
||||
{
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: self.disk_id,
|
||||
physical_slot: self.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let bytes = encode(&record)?;
|
||||
let directory = directory.to_path_buf();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
/// This caller already owns SourceRW and destination metadata ownership.
|
||||
/// An Aborting survivor can confirm a historical target only with its
|
||||
/// independent, same-disk publication anchor. Closure never creates one.
|
||||
@@ -1456,6 +1683,8 @@ struct ValidatedPreparedPublication {
|
||||
length: u64,
|
||||
}
|
||||
|
||||
type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
|
||||
|
||||
pub(super) struct NativeMultipartPublication {
|
||||
pub store: Arc<SourceStore>,
|
||||
pub publication: PublicationIdentity,
|
||||
@@ -1463,22 +1692,49 @@ pub(super) struct NativeMultipartPublication {
|
||||
state: parking_lot::Mutex<SourceState>,
|
||||
capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
|
||||
validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
|
||||
deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
pub(super) async fn prepare_native_multipart_publication(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_native_multipart_publication_deferred(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_native_multipart_publication_inner(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
defer_persist: bool,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
// Real Complete is parts-backed, including a zero-byte last part.
|
||||
if incoming.data.is_some() || incoming.inline_data() {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
|
||||
let loaded = store.load().await?;
|
||||
let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
|
||||
load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
|
||||
} else {
|
||||
store.load().await?
|
||||
};
|
||||
let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
let original = loaded
|
||||
.head
|
||||
@@ -1569,7 +1825,44 @@ impl LocalDisk {
|
||||
};
|
||||
// Copy the small source FI to an independent immutable inode. Its
|
||||
// canonical inode is never truncated to build the destination proxy.
|
||||
store.write_owned_backup_and_persist(owner, false, original, &state).await?;
|
||||
let directory = staging.private_directory(publication.final_data_dir);
|
||||
let anchor_record = NativeRetirementAnchor {
|
||||
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
|
||||
disk_id: store.disk_id,
|
||||
physical_slot: store.physical_slot,
|
||||
publication: publication.clone(),
|
||||
source_original_sha256: state.original_sha256,
|
||||
source_validation_receipt_id: state.backup_owner,
|
||||
};
|
||||
let anchor_bytes = encode(&anchor_record)?;
|
||||
state.validate(store.disk_id, &store.path)?;
|
||||
let original_bytes = original.to_vec();
|
||||
let deferred_initial = if defer_persist {
|
||||
Some((original_bytes, directory, anchor_bytes))
|
||||
} else {
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source_dir)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
None
|
||||
};
|
||||
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
|
||||
let context = NativeMultipartPublication {
|
||||
store,
|
||||
@@ -1578,14 +1871,9 @@ impl LocalDisk {
|
||||
state: parking_lot::Mutex::new(state),
|
||||
capability: parking_lot::Mutex::new(Some(capability)),
|
||||
validated_prepared: parking_lot::Mutex::new(None),
|
||||
deferred_initial: parking_lot::Mutex::new(deferred_initial),
|
||||
legacy,
|
||||
};
|
||||
let directory = staging.private_directory(context.publication.final_data_dir);
|
||||
let reserved = context.state.lock().clone();
|
||||
context
|
||||
.store
|
||||
.write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
|
||||
.await?;
|
||||
Ok(context)
|
||||
}
|
||||
}
|
||||
@@ -1594,18 +1882,15 @@ impl NativeMultipartPublication {
|
||||
pub fn canonical_head(&self) -> PathBuf {
|
||||
self.store.source.join(STORAGE_FORMAT_FILE)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if encode(current_state)? != encode(&state)? || !current.head_matches_state {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
// Validate the actual persisted representation once, after FileMeta's
|
||||
// canonicalization. No unvalidated input can install a source proxy.
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
@@ -1618,17 +1903,298 @@ impl NativeMultipartPublication {
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
self.store
|
||||
.write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
|
||||
.await?;
|
||||
} else {
|
||||
self.store.persist(&state).await?;
|
||||
}
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
let prepared_path = &self.prepared_path;
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
|
||||
if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = setup_res {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub async fn prepare_and_install_head(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
let prepared_path = self.prepared_path.clone();
|
||||
let canonical = self.canonical_head();
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(&prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
|
||||
prepared_metadata_source.write_all(&new_bytes, true)?;
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub async fn prepare_and_install_head(
|
||||
self: &Arc<Self>,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let this = Arc::clone(self);
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
|
||||
{
|
||||
let guard = self.validated_prepared.lock();
|
||||
@@ -1699,13 +2265,13 @@ impl NativeMultipartPublication {
|
||||
// return an error after consuming the last claimable source heads.
|
||||
self.store.restore_exact_target_with_lease(&state, lease).await
|
||||
}
|
||||
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let current = self.store.load().await?;
|
||||
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
if current.head.is_some() || encode(current_state)? != encode(&state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let (expected_sha256, expected_length) = {
|
||||
let guard = self.validated_prepared.lock();
|
||||
let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
@@ -1717,16 +2283,88 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
(validated.sha256, validated.length)
|
||||
};
|
||||
self.store
|
||||
.read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
let target_dir = self.store.target_path(&self.publication)?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
let snapshot = self.store.persist_with_source_sync(&state, true).await?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleaned.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
let proof = self
|
||||
.store
|
||||
@@ -1738,6 +2376,21 @@ impl NativeMultipartPublication {
|
||||
undo_capability: capability,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finish(
|
||||
self: &Arc<Self>,
|
||||
target_lease: Arc<os::NamespaceMutationLease>,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
|
||||
@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
|
||||
initial: bool,
|
||||
}
|
||||
impl MultipartSourceAdmission {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn is_initial(&self) -> bool {
|
||||
self.initial
|
||||
}
|
||||
pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
|
||||
let admission = self.clone();
|
||||
Arc::new(move || admission.validate())
|
||||
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
|
||||
logical_path: PathBuf,
|
||||
io_path: PathBuf,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
cleaned: Arc<std::sync::atomic::AtomicBool>,
|
||||
fresh_source_head: Option<bytes::Bytes>,
|
||||
}
|
||||
|
||||
impl MultipartCommitStaging {
|
||||
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
|
||||
pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
|
||||
self.io_path.join(directory.to_string())
|
||||
}
|
||||
pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
|
||||
(self.io_path.clone(), Arc::clone(&self.cleaned))
|
||||
}
|
||||
pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
|
||||
self.fresh_source_head.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MultipartCommitStaging {
|
||||
fn drop(&mut self) {
|
||||
if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let logical_path = self.logical_path.clone();
|
||||
let io_path = self.io_path.clone();
|
||||
let owner = self.owner.clone();
|
||||
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
let root = upload_scope_root(upload_or_part_path, part_path);
|
||||
let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
|
||||
return false;
|
||||
};
|
||||
super::multipart_source_state::has_clean_active_multipart_source(&source)
|
||||
}
|
||||
|
||||
pub(super) fn multipart_source_admission(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -356,12 +381,14 @@ impl LocalDisk {
|
||||
let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
|
||||
// The attempt root is exactly the namespace used by the tmp sweeper.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
|
||||
let staging = MultipartCommitStaging {
|
||||
let mut staging = MultipartCommitStaging {
|
||||
path: attempt,
|
||||
lease,
|
||||
logical_path,
|
||||
io_path,
|
||||
owner,
|
||||
cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
fresh_source_head: None,
|
||||
};
|
||||
let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
|
||||
let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
|
||||
@@ -371,31 +398,37 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
|
||||
let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
|
||||
(raw, source, true)
|
||||
} else {
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
(bytes::Bytes::from(raw), source, false)
|
||||
};
|
||||
let source_dir = source
|
||||
.data_dir
|
||||
.filter(|dir| !dir.is_nil())
|
||||
@@ -412,23 +445,31 @@ impl LocalDisk {
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut pinned_source_data = None;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
pinned_source_data =
|
||||
Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
|
||||
}
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
|
||||
};
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
.max(actual.actual_size);
|
||||
@@ -464,10 +505,15 @@ impl LocalDisk {
|
||||
cancelled,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
let has_state = !from_active_cache
|
||||
&& (source_object.join(".mpu-source-state").exists()
|
||||
|| source_object.join(".mpu-recovery").exists()
|
||||
|| source_object.join(".mpu-completions").exists());
|
||||
Ok((!has_state).then_some(raw))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
staging.fresh_source_head = fresh_head;
|
||||
Ok(staging)
|
||||
}
|
||||
}
|
||||
@@ -1219,4 +1265,211 @@ mod tests {
|
||||
.expect("non-part empty hash cleanup must remain supported");
|
||||
assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
|
||||
use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
|
||||
let (_root, disk, completed, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
|
||||
.await
|
||||
.expect("fresh multipart upload metadata");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
let read_fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("cached read_version");
|
||||
assert_eq!(read_fi.data_dir, Some(data_dir));
|
||||
|
||||
let destination = format!("{upload}/{data_dir}/part.1");
|
||||
let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part transaction");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"in-flight part transaction must mark active source as having unsettled part txn"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part transaction");
|
||||
assert!(
|
||||
disk.has_clean_active_multipart_source(&upload, false),
|
||||
"settled commit must restore clean active multipart source status"
|
||||
);
|
||||
let read_parts = disk
|
||||
.read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
|
||||
.await
|
||||
.expect("cached read_parts");
|
||||
assert_eq!(read_parts.len(), 1);
|
||||
assert_eq!(read_parts[0].number, 1);
|
||||
|
||||
// Prepare part 2, then replay Commit for already-settled part 1: part 2's
|
||||
// pending transaction must keep the upload marked as having unsettled part txns.
|
||||
let destination2 = format!("{upload}/{data_dir}/part.2");
|
||||
let mut part2_info = completed.parts[0].clone();
|
||||
part2_info.number = 2;
|
||||
let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part 2");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part 2 transaction");
|
||||
assert!(!disk.has_clean_active_multipart_source(&upload, false));
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replayed part 1 commit settlement");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"replayed settlement for part 1 must not clear part 2's pending transaction"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part 2");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part 2 transaction");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
// Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
|
||||
.await
|
||||
.expect("delete xl.meta via delete_paths");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"per-file head deletion must invalidate the canonical upload cache key"
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
|
||||
use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
|
||||
|
||||
let (_root, disk, _, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
let source_dir = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
|
||||
.expect("source dir");
|
||||
let base_dir = disk
|
||||
.io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.expect("multipart bucket path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
let init_wrapper = wrapper.clone();
|
||||
let init_upload = upload.clone();
|
||||
let init_task = tokio::spawn(async move {
|
||||
init_wrapper
|
||||
.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("initial write_metadata reaches parent dir fsync")
|
||||
.expect("entered channel");
|
||||
|
||||
let retire_wrapper = wrapper.clone();
|
||||
let retire_path = format!("{upload}/xl.meta");
|
||||
let mut retire_fut = Box::pin(async move {
|
||||
retire_wrapper
|
||||
.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
|
||||
"concurrent retirement must block on SourceRW while initial head publication is in flight"
|
||||
);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
init_task.await.expect("init join").expect("initial write_metadata succeeds");
|
||||
retire_fut
|
||||
.await
|
||||
.expect("concurrent retirement completes after SourceRW release");
|
||||
|
||||
assert!(
|
||||
!super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
|
||||
"retired source must not remain active in cache after interleaved initial publication"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
super::super::multipart_source_state::validate_source_mutation(
|
||||
&source_dir,
|
||||
&base_dir,
|
||||
&disk.publication_root,
|
||||
false
|
||||
),
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"validate_source_mutation must reject non-initial mutation on retired source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,6 +401,13 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
|
||||
Self::Remote(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+992
-199
File diff suppressed because it is too large
Load Diff
@@ -828,13 +828,27 @@ pub async fn create_bitrot_writer(
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
create_bitrot_writer_with_orig_volume(is_inline_buffer, disk, "", volume, path, length, shard_size, checksum_algo).await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn create_bitrot_writer_with_orig_volume(
|
||||
is_inline_buffer: bool,
|
||||
disk: Option<&DiskStore>,
|
||||
origvolume: &str,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
length: i64,
|
||||
shard_size: usize,
|
||||
checksum_algo: HashAlgorithm,
|
||||
) -> disk::error::Result<BitrotWriterWrapper> {
|
||||
let writer = if is_inline_buffer {
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else if let Some(disk) = disk {
|
||||
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
|
||||
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
let file = disk.create_file(origvolume, volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let file = instrument_raw_shard_writer(file, disk.is_local());
|
||||
CustomWriter::new_tokio_writer(file)
|
||||
|
||||
@@ -429,6 +429,7 @@ impl WritePlan {
|
||||
let checksum = reader.content_hash().clone();
|
||||
let trailer = reader.get_trailer().cloned();
|
||||
|
||||
let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
|
||||
let encrypted = self.encryption.is_some();
|
||||
if let Some(algorithm) = self.compression {
|
||||
reader = HashReader::from_reader(
|
||||
@@ -448,7 +449,8 @@ impl WritePlan {
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
|
||||
@@ -333,6 +333,34 @@ struct ProofCache {
|
||||
preferred: usize,
|
||||
}
|
||||
|
||||
type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
|
||||
|
||||
static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.get(&(part.generation, part.root, stripe))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
|
||||
let mut map = VERIFIED_STRIPE_RECORDS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().copied()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert((part.generation, part.root, stripe), record);
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.retain(|(stored_gen, _, _), _| *stored_gen != generation);
|
||||
}
|
||||
|
||||
impl PartProofReader {
|
||||
pub(crate) async fn reconstruction_proof(
|
||||
&self,
|
||||
@@ -410,6 +438,9 @@ impl PartProofReader {
|
||||
|
||||
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
|
||||
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
|
||||
if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let mut cached = self.cached.lock().await;
|
||||
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
|
||||
return Ok(bytes.clone());
|
||||
@@ -465,6 +496,9 @@ impl PartProofReader {
|
||||
bytes
|
||||
};
|
||||
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
if self.part.stripes <= 4 {
|
||||
put_verified_stripe_record(&self.part, stripe, record.clone());
|
||||
}
|
||||
if cached.records.len() == 2 {
|
||||
cached.records.pop_front();
|
||||
}
|
||||
@@ -863,4 +897,41 @@ mod tests {
|
||||
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
|
||||
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let inline_bytes = prepared.inline_bytes().expect("inline bytes");
|
||||
let part = prepared.part;
|
||||
|
||||
let warm_reader = PartProofReader {
|
||||
part: part.clone(),
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: Some(inline_bytes),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
|
||||
assert!(
|
||||
get_verified_stripe_record(&part, 0).is_some(),
|
||||
"small verified stripe record should populate the verified stripe cache"
|
||||
);
|
||||
|
||||
let cached_reader = Arc::new(PartProofReader {
|
||||
part,
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
|
||||
assert!(
|
||||
cached_reader.verify(0, 0, shards[1]).await.is_err(),
|
||||
"cached record must still reject corrupt or mismatched shard payloads"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1543,6 +1543,8 @@ pub struct ObjectInfo {
|
||||
/// published by the same upload. Callers must not repeat quota, replication,
|
||||
/// lifecycle, or object-created side effects for that response.
|
||||
pub multipart_completion_replayed: bool,
|
||||
/// Previous current version's size observed during `rename_data` commit.
|
||||
pub backfilled_old_current_size: Option<crate::disk::OldCurrentSize>,
|
||||
}
|
||||
|
||||
impl Default for ObjectInfo {
|
||||
@@ -1585,6 +1587,7 @@ impl Default for ObjectInfo {
|
||||
replication_decision: String::new(),
|
||||
checksum: None,
|
||||
multipart_completion_replayed: false,
|
||||
backfilled_old_current_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1628,6 +1631,7 @@ impl Clone for ObjectInfo {
|
||||
replication_decision: self.replication_decision.clone(),
|
||||
checksum: self.checksum.clone(),
|
||||
multipart_completion_replayed: self.multipart_completion_replayed,
|
||||
backfilled_old_current_size: self.backfilled_old_current_size,
|
||||
expires: self.expires,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -941,6 +941,32 @@ impl InstanceContext {
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
|
||||
///
|
||||
/// Called from `ECStore::drop`. When test fixtures share the bootstrap
|
||||
/// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
|
||||
/// reference cycle; once the store drops the weak is dangling and any
|
||||
/// `object_store()` call panics. Clearing the slot here lets the next
|
||||
/// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
|
||||
pub fn clear_bucket_metadata_sys_if_stale(&self) {
|
||||
let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
|
||||
if let Some(sys) = guard.as_ref() {
|
||||
// `read()` may block briefly; the async RwLock is acceptable here
|
||||
// because Drop is sync and the lock is held only long enough to
|
||||
// check the weak reference.
|
||||
let is_stale = {
|
||||
let s = sys.try_read();
|
||||
match s {
|
||||
Ok(s) => s.is_store_stale(),
|
||||
Err(_) => true, // contended: treat as stale-safe to clear
|
||||
}
|
||||
};
|
||||
if is_stale {
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Set this instance's background-services cancellation token (once).
|
||||
pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
|
||||
self.background_cancel_token.set(token)
|
||||
|
||||
@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
|
||||
match rebalance_error_source(err) {
|
||||
Error::Lock(err) => is_rebalance_transient_lock_error(err),
|
||||
// No metadata was written when lock admission failed. Each attempt
|
||||
// must acquire its quorum again before loading or saving this run.
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket,
|
||||
object,
|
||||
..
|
||||
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return true;
|
||||
@@ -385,9 +400,7 @@ where
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(err) => {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1)
|
||||
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
|
||||
{
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -719,6 +732,82 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
|
||||
let mut attempts = 0;
|
||||
retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(if attempts == 1 {
|
||||
resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)),
|
||||
"stop_rebalance",
|
||||
)
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("a later quorum-protected admission must be allowed to succeed");
|
||||
assert_eq!(attempts, 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
|
||||
for max_attempts in [0, 1, 3] {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, max_attempts, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)))
|
||||
})
|
||||
.await;
|
||||
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
|
||||
assert_eq!(attempts, max_attempts.max(1));
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&error),
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(if attempts == 1 {
|
||||
rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)
|
||||
} else {
|
||||
Error::OperationCanceled
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
|
||||
assert!(matches!(result, Err(Error::OperationCanceled)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
|
||||
for err in [
|
||||
@@ -731,6 +820,20 @@ mod error_source_tests {
|
||||
required: 3,
|
||||
achieved: 2,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: "data-bucket".to_string(),
|
||||
object: REBAL_META_NAME.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
|
||||
object: "other-metadata.bin".to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::other("stale rebalance run rejected: lock acquisition timed out"),
|
||||
Error::other("rebalance distributed run fence lost"),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
|
||||
@@ -5340,6 +5340,14 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
|
||||
@@ -5539,6 +5547,14 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
@@ -5598,10 +5614,11 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
let part = ObjectPartInfo::unmarshal(&meta)?;
|
||||
let integrity = part.integrity.map(Arc::new);
|
||||
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
@@ -5623,7 +5640,9 @@ impl SetDisks {
|
||||
if let Err(error) = prepared {
|
||||
return Some(Err(error));
|
||||
}
|
||||
if let Some(integrity) = integrity {
|
||||
if let Some(integrity) = integrity
|
||||
&& !disk.is_local()
|
||||
{
|
||||
let Some((directory, _)) = dst_object.rsplit_once('/') else {
|
||||
return Some(Err(DiskError::FileCorrupt));
|
||||
};
|
||||
@@ -5740,7 +5759,36 @@ impl SetDisks {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
let all_prepared_succeeded = errs
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
|
||||
let committed = if all_prepared_succeeded {
|
||||
let settles = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Commit)
|
||||
.await
|
||||
}
|
||||
});
|
||||
let settle_results = join_all(settles).await;
|
||||
if settle_results.iter().filter(|res| res.is_ok()).count() >= write_quorum
|
||||
&& settle_results
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(res, disk)| disk.is_none() || res.is_ok())
|
||||
{
|
||||
true
|
||||
} else {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?
|
||||
}
|
||||
} else {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?
|
||||
};
|
||||
if !committed {
|
||||
let err = DiskError::ErasureWriteQuorum;
|
||||
if let Some(context) = quorum_context {
|
||||
@@ -6737,9 +6785,7 @@ impl SetDisks {
|
||||
object,
|
||||
);
|
||||
}
|
||||
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
||||
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
||||
if should_prevent_write(&oi, if_none_match, if_match)
|
||||
if should_prevent_write(&oi, http_preconditions.if_none_match_value(), http_preconditions.if_match_value())
|
||||
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
|
||||
{
|
||||
return Some(StorageError::PreconditionFailed);
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for parts in versions.into_values() {
|
||||
for (version_id, parts) in versions {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,15 +173,25 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
|
||||
@@ -80,7 +80,7 @@ use crate::erasure::codec::bridge::{
|
||||
use crate::erasure::coding;
|
||||
use crate::error::{Error, Result, is_err_version_not_found};
|
||||
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer, create_bitrot_writer_with_orig_volume};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::get_object_body_cache_hook;
|
||||
use crate::object_api::object_api_utils::get_raw_etag;
|
||||
@@ -120,7 +120,6 @@ use bytesize::ByteSize;
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use regex::Regex;
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_filemeta::{
|
||||
@@ -4697,6 +4696,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
// Reusing a held writer additionally requires the same quorum, so repeated
|
||||
// clients must not give different weight to an otherwise identical domain.
|
||||
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
!left.is_empty()
|
||||
&& left.len() == right.len()
|
||||
&& left.iter().all(|client| {
|
||||
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
})
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -5646,9 +5656,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
// Distributed lock clients receive the raw ObjectKey; the pool
|
||||
// namespace is not part of the key stored by their backends.
|
||||
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -8036,16 +8046,29 @@ fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Opti
|
||||
Some(&part_numbers[index..])
|
||||
}
|
||||
|
||||
pub fn canonicalize_etag(etag: &str) -> String {
|
||||
let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
|
||||
re.replace_all(etag, "$1").to_string()
|
||||
fn canonicalized_etag(etag: &str) -> Cow<'_, str> {
|
||||
// Preserve the legacy suffix-matching behavior for embedded quotes.
|
||||
let trimmed = etag.trim_end_matches('"');
|
||||
let Some(last_quote) = trimmed.rfind('"') else {
|
||||
return Cow::Borrowed(trimmed);
|
||||
};
|
||||
let prefix = trimmed[..last_quote].trim_end_matches('"');
|
||||
let suffix = &trimmed[last_quote + 1..];
|
||||
if prefix.is_empty() {
|
||||
Cow::Borrowed(suffix)
|
||||
} else {
|
||||
let mut result = String::with_capacity(prefix.len() + suffix.len());
|
||||
result.push_str(prefix);
|
||||
result.push_str(suffix);
|
||||
Cow::Owned(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
|
||||
if condition.trim() == "*" {
|
||||
return true;
|
||||
}
|
||||
canonicalize_etag(etag) == canonicalize_etag(condition)
|
||||
canonicalized_etag(etag) == canonicalized_etag(condition)
|
||||
}
|
||||
|
||||
fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
@@ -8060,15 +8083,12 @@ fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
.or_else(|| trimmed.strip_prefix("w/"))
|
||||
.unwrap_or(trimmed)
|
||||
}
|
||||
canonicalize_etag(strip_weak(etag)) == canonicalize_etag(strip_weak(condition))
|
||||
canonicalized_etag(strip_weak(etag)) == canonicalized_etag(strip_weak(condition))
|
||||
}
|
||||
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
|
||||
let if_none_match = if_none_match
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.as_deref().map(str::trim).filter(|condition| !condition.is_empty());
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<&str>, if_match: Option<&str>) -> bool {
|
||||
let if_none_match = if_none_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
|
||||
match &oi.etag {
|
||||
Some(etag) => {
|
||||
@@ -9029,6 +9049,13 @@ mod tests {
|
||||
"constructing an object lock must not clone each client Arc"
|
||||
);
|
||||
|
||||
// Check shared ownership before acquisition can spawn pending cleanup.
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
|
||||
let write_error = write_lock
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9043,11 +9070,6 @@ mod tests {
|
||||
),
|
||||
"the shared client representation must preserve the exact write quorum result: {write_error}"
|
||||
);
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
let read_guard = read_lock
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9871,7 +9893,7 @@ mod tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
@@ -9910,11 +9932,25 @@ mod tests {
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pools using identical backends lock the same raw object key"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
|
||||
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
|
||||
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
|
||||
assert!(!same_distributed_lock_quorum(
|
||||
&[a.clone(), a.clone(), b.clone()],
|
||||
&[a.clone(), b.clone(), b]
|
||||
));
|
||||
assert!(!same_distributed_lock_quorum(&[], &[]));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
@@ -13772,6 +13808,22 @@ mod tests {
|
||||
assert!(result2.iter().all(|d| d.is_none()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalized_etag_preserves_legacy_regex_semantics() {
|
||||
let regex = regex::Regex::new("\"*?([^\"]*?)\"*?$").expect("valid legacy ETag pattern");
|
||||
let alphabet = ['a', '"', '\n', 'é', '\0'];
|
||||
for length in 0..=6_u32 {
|
||||
for mut number in 0..alphabet.len().pow(length) {
|
||||
let mut input = String::new();
|
||||
for _ in 0..length {
|
||||
input.push(alphabet[number % alphabet.len()]);
|
||||
number /= alphabet.len();
|
||||
}
|
||||
assert_eq!(canonicalized_etag(&input), regex.replace_all(&input, "$1"), "input: {input:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etag_matches() {
|
||||
assert!(e_tag_matches("abc", "abc"));
|
||||
@@ -14347,61 +14399,61 @@ mod tests {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("abc".to_string());
|
||||
let if_none_match = Some("abc");
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("def".to_string());
|
||||
let if_match = Some("def");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("*".to_string());
|
||||
let if_match = Some("*");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("*".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("*");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: None,
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some(String::new());
|
||||
let if_match = Some(" ".to_string());
|
||||
let if_none_match = Some("");
|
||||
let if_match = Some(" ");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\"".to_string()), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\"".to_string()), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"".to_string())));
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\""), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\""), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -41,8 +41,8 @@ use super::super::{
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
|
||||
create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
|
||||
get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
|
||||
get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
|
||||
log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
|
||||
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
|
||||
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
|
||||
@@ -124,9 +124,10 @@ async fn create_part_writers(
|
||||
let Some(disk) = disk else {
|
||||
return (None, Some(DiskError::DiskNotFound));
|
||||
};
|
||||
match create_bitrot_writer(
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
false,
|
||||
Some(disk),
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
path,
|
||||
length,
|
||||
@@ -977,7 +978,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() {
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1002,6 +1003,13 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.into_iter()
|
||||
.map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
|
||||
.filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.map(|sha_dir| {
|
||||
if root_prefix.is_empty() {
|
||||
sha_dir
|
||||
} else {
|
||||
format!("{root_prefix}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let listed_upload_dirs = stream::iter(sha_dirs)
|
||||
.map(|sha_dir| {
|
||||
@@ -1124,6 +1132,7 @@ impl SetDisks {
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -1165,7 +1174,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
|
||||
let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
|
||||
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
|
||||
}
|
||||
|
||||
@@ -1187,7 +1197,7 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
@@ -1200,8 +1210,9 @@ impl SetDisks {
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
let (_, candidate_paths, _, has_minority_candidate) = self
|
||||
.discover_multipart_upload_paths(bucket, object, &expected_parent, false)
|
||||
.await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::DecommissionCapacityBlocked {
|
||||
message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
|
||||
@@ -1750,6 +1761,54 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
// Every disk must confirm absence in both namespaces before cleanup can
|
||||
// remove catalog protection from the bucket.
|
||||
for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let (disks, paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for upload_path in paths {
|
||||
let (sha_dir, _) = upload_path
|
||||
.rsplit_once('/')
|
||||
.filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
|
||||
.ok_or(DiskError::CorruptedFormat)?;
|
||||
let (metadata, errors) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
|
||||
.await?;
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
|
||||
let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
|
||||
let (Some(owner_bucket), Some(owner_object)) = (
|
||||
file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) else {
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
};
|
||||
if owner_bucket.is_empty()
|
||||
|| owner_object.is_empty()
|
||||
|| Self::get_multipart_sha_dir(owner_bucket, owner_object)
|
||||
!= sha_dir
|
||||
.strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.unwrap_or(sha_dir)
|
||||
{
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
if owner_bucket == bucket {
|
||||
return Err(Error::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1761,7 +1820,8 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -2372,7 +2432,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let multipart_part_size = data.size();
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
|
||||
);
|
||||
|
||||
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
|
||||
@@ -2436,7 +2496,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
let part_integrity = if let Some(integrity) = integrity {
|
||||
Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
|
||||
Some(
|
||||
integrity
|
||||
.write(
|
||||
&mut shuffle_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else { None };
|
||||
if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
@@ -3471,10 +3540,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
bind_multipart_proof_snapshot(&disks, &self.multipart_proof_verification_context())?;
|
||||
|
||||
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let part_meta_paths = uploaded_parts
|
||||
.iter()
|
||||
.map(|v| format!("{part_path}part.{0}.meta", v.part_num))
|
||||
@@ -3482,6 +3547,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
|
||||
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let object_parts = Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
@@ -4389,6 +4458,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let needs_immediate_heal = rename_commit.needs_immediate_heal();
|
||||
let op_old_dir = rename_commit.data_dir;
|
||||
let cleanup_disks = rename_commit.cleanup_disks;
|
||||
let old_current_size = rename_commit.old_current_size;
|
||||
let committed_file_info = rename_commit.committed_file_info;
|
||||
|
||||
if needs_immediate_heal {
|
||||
@@ -4457,7 +4527,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
let can_skip_part_cleanup = !needs_upgrade
|
||||
&& commit_disks
|
||||
.iter()
|
||||
.all(|disk| disk.as_ref().is_some_and(|disk| disk.is_local()));
|
||||
if !can_skip_part_cleanup {
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
}
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
@@ -4503,7 +4579,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
drop(_upload_guard.take());
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
|
||||
let mut obj_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned);
|
||||
obj_info.backfilled_old_current_size = old_current_size;
|
||||
Ok(obj_info)
|
||||
};
|
||||
|
||||
let result = if detach_commit_owner {
|
||||
@@ -9982,7 +10060,7 @@ mod tests {
|
||||
|
||||
// Real disk callbacks and health probes must share wall-clock deadlines.
|
||||
// Advancing Tokio time can expire unrelated healthy-drive probes.
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
|
||||
let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "mpu-quorum-success-physical";
|
||||
@@ -10009,6 +10087,8 @@ mod tests {
|
||||
local.observe_multipart_cleanup_for_test(&upload_path)
|
||||
}).collect::<Vec<_>>();
|
||||
let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
|
||||
// Fault only the held leg; healthy disks keep their normal I/O budgets.
|
||||
let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
|
||||
let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
|
||||
let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -10165,7 +10245,15 @@ mod tests {
|
||||
let size = copy_info.get_actual_size().expect("actual source size");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
|
||||
tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
|
||||
.await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
|
||||
.await.expect("cross-key Copy cannot wait on the old successful logical owner")
|
||||
.unwrap_or_else(|error| {
|
||||
let health = disks.iter().enumerate().map(|(slot, disk)| {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
|
||||
let metrics = local.metrics_snapshot();
|
||||
(slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
|
||||
}).collect::<Vec<_>>();
|
||||
panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
|
||||
});
|
||||
drop(copy_info);
|
||||
let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
|
||||
let mut actual = Vec::new();
|
||||
@@ -11194,6 +11282,163 @@ mod tests {
|
||||
assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-target";
|
||||
let other = "multipart-cleanup-other";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create target bucket");
|
||||
disk.make_volume(other).await.expect("create other bucket");
|
||||
}
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an empty upload namespace permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(other, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in other bucket");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an upload owned by another bucket permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in target bucket");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-offline";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks.disks.write().await[3] = None;
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing preserves its discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the offline disk must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-list-error";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
|
||||
let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
|
||||
tokio::fs::create_dir_all(&multipart_dir)
|
||||
.await
|
||||
.expect("create multipart staging directory");
|
||||
tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
|
||||
.await
|
||||
.expect("replace one upload parent directory with a file");
|
||||
multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
|
||||
.await
|
||||
.expect_err("the affected disk must fail directory enumeration");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing still accepts the remaining discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
|
||||
.await
|
||||
.expect("remove the invalid directory entry");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after repair");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_data_movement_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-data-movement";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
|
||||
);
|
||||
set_disks
|
||||
.new_multipart_upload(
|
||||
bucket,
|
||||
"object.bin",
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create data movement upload");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_unreadable_ownership() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-corrupt";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let mut corrupted = 0;
|
||||
for temp_dir in &temp_dirs {
|
||||
for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
tokio::fs::write(path, b"invalid upload metadata")
|
||||
.await
|
||||
.expect("corrupt upload metadata");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "upload metadata must exist before corruption");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("unreadable upload ownership must block cleanup");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
|
||||
@@ -46,9 +46,9 @@ use super::super::{
|
||||
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
|
||||
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
|
||||
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, delete_file_info_version_id,
|
||||
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
|
||||
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer_with_orig_volume, debug,
|
||||
delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
|
||||
finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
|
||||
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
|
||||
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
|
||||
@@ -4459,9 +4459,10 @@ impl SetDisks {
|
||||
if let Some(disk) = disk_op
|
||||
&& disk.is_online().await
|
||||
{
|
||||
match create_bitrot_writer(
|
||||
match create_bitrot_writer_with_orig_volume(
|
||||
is_inline_buffer,
|
||||
Some(disk),
|
||||
bucket,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_obj,
|
||||
shard_file_size,
|
||||
@@ -4533,7 +4534,7 @@ impl SetDisks {
|
||||
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
|
||||
);
|
||||
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
|
||||
@@ -6128,7 +6129,11 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if !tail_owns_tmp_cleanup {
|
||||
let all_local_disks_cleaned_tmp = all_commit_disks_local
|
||||
&& online_disks.len() == commit_set.set_drive_count
|
||||
&& online_disks.iter().all(Option::is_some)
|
||||
&& !issue3031_diag_enabled();
|
||||
if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
|
||||
let cleanup_set = commit_set.clone();
|
||||
let cleanup_tmp_dir = commit_tmp_dir.clone();
|
||||
cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
|
||||
@@ -11064,7 +11069,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if let Some(err) = &gerr
|
||||
&& goi.name.is_empty()
|
||||
{
|
||||
if should_force_delete_marker_for_missing_version(&opts) {
|
||||
if (is_err_object_not_found(err) || is_err_version_not_found(err))
|
||||
&& should_force_delete_marker_for_missing_version(&opts)
|
||||
{
|
||||
version_found = false;
|
||||
} else {
|
||||
if is_err_object_not_found(err) {
|
||||
|
||||
@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
|
||||
WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
|
||||
WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -523,7 +523,22 @@ impl ECStore {
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
|
||||
}
|
||||
|
||||
/// Validate an external fence and the bucket incarnation in one metadata transaction.
|
||||
pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
|
||||
@@ -3692,6 +3692,177 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// The RPC lock service uses the raw ObjectKey and one backend per node,
|
||||
// even when the requesting sets carry different pool namespace strings.
|
||||
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
|
||||
use rustfs_lock::client::{LockClient, local::LocalClient};
|
||||
|
||||
let clients = || {
|
||||
(0..2)
|
||||
.map(|_| {
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let first = clients();
|
||||
let second = if shared { first.clone() } else { clients() };
|
||||
let mut pools = source.pools.clone();
|
||||
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
|
||||
for set in &mut Arc::make_mut(pool).disk_set {
|
||||
Arc::make_mut(set).lockers = lockers.clone();
|
||||
}
|
||||
}
|
||||
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
||||
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
|
||||
Arc::new(ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: Default::default(),
|
||||
pools,
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(state),
|
||||
ctx: Arc::clone(&source.ctx),
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
|
||||
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let first_lock = first
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("first pool lock");
|
||||
let second_lock = second
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("second pool lock");
|
||||
let guard = first_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"different pool namespaces still conflict on the same backend and object key"
|
||||
);
|
||||
drop(guard);
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(3))
|
||||
.await
|
||||
.expect("second writer should acquire after the first releases");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let missing = remove_pool_meta_shard(&store, 1).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cross-pool repair must not reenter the shared writer")
|
||||
.expect("pool metadata repair should return a result");
|
||||
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
assert!(
|
||||
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
|
||||
"the selected missing shard must be restored"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
|
||||
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let lock = target
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("target lock");
|
||||
let guard = lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("hold target lock");
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let heal_store = Arc::clone(&store);
|
||||
let mut heal = tokio::spawn(async move {
|
||||
heal_store
|
||||
.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
barrier.wait_until_external_heal_target_lock_attempted(),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must attempt the independent target lock");
|
||||
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
|
||||
drop(guard);
|
||||
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
|
||||
.await
|
||||
.expect("inspection should progress after target lock release")
|
||||
.expect("heal task should not panic")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
|
||||
|
||||
@@ -5340,6 +5340,203 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn conditional_multipart_must_not_commit_against_an_older_readable_pool() {
|
||||
use crate::storage_api_contracts::multipart::CompletePart;
|
||||
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create conditional multipart test directory");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "conditional-mpu-unreadable", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("conditional-mpu-{}", uuid::Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create conditional multipart test bucket");
|
||||
let mut old_body = PutObjReader::from_vec(b"older-pool".to_vec());
|
||||
let old = store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write old object to staging pool");
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload in pool 0");
|
||||
let mut part_body = PutObjReader::from_vec(b"stale-writer".to_vec());
|
||||
let part = store.pools[0]
|
||||
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart part in pool 0");
|
||||
let mut new_body = PutObjReader::from_vec(b"newer-pool".to_vec());
|
||||
let new = store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut new_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer object to pool 1");
|
||||
let old_etag = old.etag.clone().expect("old object must have an ETag");
|
||||
let before = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
|
||||
let mut metadata = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let path = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/xl.meta"));
|
||||
let original = tokio::fs::read(&path)
|
||||
.await
|
||||
.expect("save original object metadata before fault injection");
|
||||
metadata.push((path, original));
|
||||
}
|
||||
let mut fault_error = None;
|
||||
for (path, _) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, b"invalid object metadata").await {
|
||||
fault_error = Some(err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut fault_contents = Vec::new();
|
||||
if fault_error.is_none() {
|
||||
for (path, _) in &metadata {
|
||||
fault_contents.push(tokio::fs::read(path).await);
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
}
|
||||
let unreadable = if fault_error.is_none() {
|
||||
Some(
|
||||
store.pools[1]
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed = if fault_error.is_none() {
|
||||
let completion_store = Arc::clone(&store);
|
||||
let completion_bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let part_etag = part.etag.clone();
|
||||
Some(
|
||||
tokio::spawn(async move {
|
||||
completion_store
|
||||
.complete_multipart_upload(
|
||||
&completion_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part_etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(old_etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut restore_error = None;
|
||||
for (path, original) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, original).await {
|
||||
restore_error = Some(err);
|
||||
}
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
let after = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
let recovered_body = match store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(mut reader) => {
|
||||
let mut body = Vec::new();
|
||||
let result = reader
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.map(|_| body)
|
||||
.map_err(|err| err.to_string());
|
||||
drop(reader);
|
||||
result
|
||||
}
|
||||
Err(err) => Err(err.to_string()),
|
||||
};
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
|
||||
assert!(fault_error.is_none(), "failed to inject metadata fault: {fault_error:?}");
|
||||
assert!(restore_error.is_none(), "failed to restore metadata: {restore_error:?}");
|
||||
assert_eq!(fault_contents.len(), 4, "all four pool 1 metadata files must be inspected");
|
||||
for observed in fault_contents {
|
||||
assert_eq!(
|
||||
observed.expect("read injected metadata"),
|
||||
b"invalid object metadata",
|
||||
"each pool 1 metadata file must contain the injected corruption"
|
||||
);
|
||||
}
|
||||
assert_eq!(before.expect("latest object before fault").etag, new.etag);
|
||||
assert_ne!(old.etag, new.etag, "fixture bodies must produce different ETags");
|
||||
let read_error = unreadable
|
||||
.expect("metadata fault should have been injected")
|
||||
.expect_err("the newer pool must fail its object metadata read during the fault");
|
||||
assert!(
|
||||
!crate::error::is_err_object_not_found(&read_error) && !crate::error::is_err_version_not_found(&read_error),
|
||||
"corrupt metadata must produce a non-not-found read error: {read_error:?}"
|
||||
);
|
||||
let completed = completed
|
||||
.expect("completion should have been attempted")
|
||||
.expect("completion task should not panic");
|
||||
let Err(completion_error) = completed else {
|
||||
panic!("stale If-Match must not commit while the newer object's metadata is unreadable");
|
||||
};
|
||||
assert_eq!(completion_error, read_error, "completion must propagate the unreadable pool's error");
|
||||
assert_eq!(after.expect("newer object after recovery").etag, new.etag);
|
||||
assert_eq!(recovered_body.expect("newer body after recovery"), b"newer-pool");
|
||||
let parts = parts.expect("failed completion must leave the upload retryable");
|
||||
assert_eq!(parts.parts.len(), 1);
|
||||
assert_eq!(parts.parts[0].etag, part.etag);
|
||||
}
|
||||
|
||||
async fn migrate_versioned_decommission_test_object(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -37,8 +37,8 @@ use crate::error::{
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::list::{
|
||||
ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
|
||||
StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -80,6 +80,21 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthoritativeListOperations for ECStore {
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
self.walk_authoritatively(rx, bucket, prefix, result, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the versioned listing options used by recursive object walks.
|
||||
///
|
||||
/// `list_path_result` normally derives these scan bounds before dispatching to
|
||||
@@ -1292,6 +1307,37 @@ impl ECStore {
|
||||
);
|
||||
|
||||
let rx = rx.child_token();
|
||||
if self.pools.len() == 1 && self.pools[0].disk_set.len() == 1 {
|
||||
rustfs_io_metrics::record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 1, 1);
|
||||
let res = self.pools[0].disk_set[0].list_path(rx, opts, sender).await;
|
||||
let mut outcome = ListPathRawOutcome::default();
|
||||
let err = match res {
|
||||
Ok(set_outcome) => {
|
||||
outcome.limit_reached = set_outcome.limit_reached;
|
||||
None
|
||||
}
|
||||
Err(err) => Some(err),
|
||||
};
|
||||
let errs = [err];
|
||||
if is_all_not_found(&errs) {
|
||||
return Ok(outcome);
|
||||
}
|
||||
if let Some(err) = errs.into_iter().flatten().next() {
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"store_list_objects_list_merged",
|
||||
merge_started.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
debug!(
|
||||
set_pair_count = 1,
|
||||
limit_reached = outcome.limit_reached,
|
||||
error_count = 0,
|
||||
"store list_merged finished"
|
||||
);
|
||||
return Ok(outcome);
|
||||
}
|
||||
|
||||
let mut futures = Vec::new();
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
@@ -1519,6 +1565,57 @@ impl ECStore {
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn walk_authoritatively(
|
||||
self: Arc<Self>,
|
||||
cancellation: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
let cancel = cancellation.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let mut scans = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
inputs.push(receiver);
|
||||
let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
|
||||
// Internal absence checks must include reserved object prefixes.
|
||||
scan.skip_hidden_prefix_check = true;
|
||||
scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
|
||||
}
|
||||
}
|
||||
if scans.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
|
||||
let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
|
||||
let scan_results = join_all(scans).await;
|
||||
let merge_result = merge.await.map_err(Error::from)?;
|
||||
let forward_result = forwarder.await.map_err(Error::from);
|
||||
|
||||
// An unavailable set cannot establish absence, even when another set
|
||||
// completed. Only an explicitly missing directory or volume is empty.
|
||||
for scan in scan_results {
|
||||
match scan {
|
||||
Ok(outcome) if !outcome.limit_reached => {}
|
||||
Ok(_) => return Err(StorageError::ErasureReadQuorum),
|
||||
Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
merge_result?;
|
||||
forward_result?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(Error::other("authoritative object walk was cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
@@ -1811,7 +1908,7 @@ async fn gather_results(
|
||||
if opts.versioned
|
||||
&& is_object
|
||||
&& entry
|
||||
.xl_meta()
|
||||
.xl_meta_ref()
|
||||
.is_ok_and(|meta| !meta.versions.is_empty() && meta.versions.iter().all(|version| version.header.free_version()))
|
||||
{
|
||||
continue;
|
||||
@@ -2489,7 +2586,8 @@ impl SetDisks {
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
/// version is resolved at the read quorum of its own erasure layout; a
|
||||
/// version uses the read quorum of its own erasure layout. Null versions
|
||||
/// in the system bucket also retain GET's commit-proof selection. A
|
||||
/// version that cannot be ruled out fails the scan instead of being
|
||||
/// omitted. Merged `ListObjectVersions` pages go through `list_path`.
|
||||
async fn list_versions_authoritatively(
|
||||
@@ -2828,6 +2926,7 @@ mod test {
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
|
||||
use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use bytes::Bytes;
|
||||
@@ -5688,6 +5787,126 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
for new_etag in ["new", "old"] {
|
||||
let new =
|
||||
test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
|
||||
let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
|
||||
|
||||
assert!(matches!(
|
||||
SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_selects_committed_null_version() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
|
||||
.collect(),
|
||||
);
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
|
||||
.expect("committed generation should resolve")
|
||||
.expect("committed record should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (old_copies, new_copies, expected) in [
|
||||
(2, 1, Some("old")),
|
||||
(1, 2, None),
|
||||
(0, 3, Some("new")),
|
||||
(0, 2, Some("new")),
|
||||
(0, 1, None),
|
||||
] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| {
|
||||
if slot < old_copies {
|
||||
Some(old.clone())
|
||||
} else if slot < old_copies + new_copies {
|
||||
Some(new.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let errors = (0..4)
|
||||
.map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
|
||||
.collect::<Vec<_>>();
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
|
||||
|
||||
if let Some(expected) = expected {
|
||||
let info = resolved
|
||||
.expect("degraded committed record should resolve")
|
||||
.expect("degraded committed record should be present")
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
} else {
|
||||
assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for old_copies in [1, 2] {
|
||||
let entries = MetaCacheEntries(vec![
|
||||
Some(new.clone()),
|
||||
Some(new.clone()),
|
||||
Some(old.clone()),
|
||||
(old_copies == 2).then(|| old.clone()),
|
||||
]);
|
||||
let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
|
||||
.expect("user bucket read quorum should resolve")
|
||||
.expect("user bucket record should be present");
|
||||
let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
|
||||
let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
|
||||
let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
|
||||
.expect("explicit version read quorum should resolve")
|
||||
.expect("explicit version should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
|
||||
let entry = test_object_meta_entry_with_erasure_versions(
|
||||
@@ -6029,6 +6248,49 @@ mod test {
|
||||
Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
|
||||
let (_dirs, store) = single_set_store(16).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for (slot, disk) in set.disks.read().await.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("fixture drive is online");
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{name}/{STORAGE_FORMAT_FILE}"),
|
||||
versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
|
||||
)
|
||||
.await
|
||||
.expect("persist reserved metadata versions");
|
||||
}
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let walk = store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let collect = async {
|
||||
let mut versions = Vec::new();
|
||||
while let Some(item) = receiver.recv().await {
|
||||
assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
|
||||
let item = item.item.expect("walk must return an object");
|
||||
versions.push((item.name, item.version_id));
|
||||
}
|
||||
versions
|
||||
};
|
||||
let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
|
||||
.await
|
||||
.expect("authoritative reserved-prefix walk must terminate");
|
||||
walk.expect("authoritative walk must succeed");
|
||||
assert_eq!(
|
||||
versions,
|
||||
[
|
||||
(name.to_string(), Some(Uuid::from_u128(1))),
|
||||
(name.to_string(), Some(Uuid::from_u128(2)))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// A versioned DeleteObject(versionId) can reach write quorum while three
|
||||
// drives are offline; they keep the deleted version after they return. A
|
||||
// different drive going offline later must not turn that sub-quorum
|
||||
@@ -6099,6 +6361,16 @@ mod test {
|
||||
.await
|
||||
.expect("set walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
let walk =
|
||||
store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let drain = async { while receiver.recv().await.is_some() {} };
|
||||
let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
|
||||
.await
|
||||
.expect("authoritative store walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let recursive_delete_scan = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)
|
||||
|
||||
@@ -498,6 +498,17 @@ pub struct ECStore {
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
}
|
||||
|
||||
impl Drop for ECStore {
|
||||
/// Clear the instance context's `BucketMetadataSys` when its backing store
|
||||
/// is dropped. Without this, the shared bootstrap context retains a
|
||||
/// `BucketMetadataSys` whose `Weak<ECStore>` is dangling after the store
|
||||
/// goes away (e.g. between sequential test fixtures that share the
|
||||
/// bootstrap), and the next `object_store()` call panics.
|
||||
fn drop(&mut self) {
|
||||
self.ctx.clear_bucket_metadata_sys_if_stale();
|
||||
}
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
|
||||
|
||||
@@ -19,10 +19,28 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_utils::path::encode_dir_object_ref;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
|
||||
type Error = Error;
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
let (_, lifecycle_guard) = self
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await?;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.ensure_no_multipart_uploads(bucket).await?;
|
||||
}
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
|
||||
std::sync::OnceLock::new();
|
||||
@@ -612,7 +630,7 @@ impl ECStore {
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts).await
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts, false).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1154,12 +1172,31 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option<StorageError> {
|
||||
async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Option<StorageError> {
|
||||
let mut lookup_opts = opts.clone();
|
||||
let http_preconditions = lookup_opts.http_preconditions?;
|
||||
lookup_opts.http_preconditions = None;
|
||||
lookup_opts.no_lock = true;
|
||||
let oi = self.get_object_info_for_delete(bucket, object, &lookup_opts).await;
|
||||
let oi = if require_all_pools {
|
||||
lookup_opts.metadata_cache_safe = false;
|
||||
lookup_opts.suppress_read_repair = true;
|
||||
// Only completion uses this branch, after validating bucket and object.
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
|
||||
.await
|
||||
.and_then(|(info, _)| {
|
||||
lookup_opts.precondition_check(&info)?;
|
||||
Ok(info)
|
||||
})
|
||||
} else {
|
||||
self.get_object_info_for_delete(bucket, object, &lookup_opts).await
|
||||
};
|
||||
crate::set_disk::SetDisks::evaluate_write_precondition(oi, &http_preconditions, bucket, object, opts)
|
||||
}
|
||||
|
||||
@@ -1186,11 +1223,13 @@ impl ECStore {
|
||||
.await;
|
||||
return enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await;
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||
{
|
||||
return Err(err);
|
||||
if opts.http_preconditions.is_some() && !opts.data_movement {
|
||||
// Preserve explicitly scoped internal reads and borrowed-lock callers.
|
||||
let require_all_pools =
|
||||
!opts.replication_request && !opts.no_lock && !opts.skip_decommissioned && !opts.skip_rebalancing;
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts, require_all_pools).await {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
let result = match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
|
||||
|
||||
@@ -5637,13 +5637,11 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if self.single_pool()
|
||||
&& !creates_latest_marker
|
||||
&& !opts.data_movement
|
||||
&& opts.http_preconditions.is_none()
|
||||
&& !opts.replication_request
|
||||
&& !opts.versioned
|
||||
&& !opts.version_suspended
|
||||
&& opts.version_id.is_none()
|
||||
&& !opts.delete_marker
|
||||
&& !self.is_suspended(0).await
|
||||
&& !self.is_pool_rebalancing(0).await
|
||||
{
|
||||
@@ -5655,6 +5653,10 @@ impl ECStore {
|
||||
.await
|
||||
{
|
||||
Ok(mut res) => {
|
||||
#[cfg(test)]
|
||||
if creates_latest_marker {
|
||||
pause_versioned_delete_marker_after_commit(bucket, object).await;
|
||||
}
|
||||
res.name = decode_dir_object(res.name.as_str());
|
||||
return Ok(res);
|
||||
}
|
||||
|
||||
@@ -837,7 +837,29 @@ impl ECStore {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Observe every pool for a conditional-write preflight. This does not
|
||||
/// hold the namespace locks needed to prove a condition at publication.
|
||||
pub(super) async fn get_latest_object_info_for_conditional_write(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, true).await
|
||||
}
|
||||
|
||||
async fn get_latest_object_info_with_pool_policy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned && !require_all_pools {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
Some(
|
||||
(0..self.pools.len())
|
||||
@@ -853,7 +875,7 @@ impl ECStore {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
|
||||
if opts.skip_rebalancing && !require_all_pools && self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -873,6 +895,9 @@ impl ECStore {
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
if require_all_pools && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
|
||||
return Err(e);
|
||||
}
|
||||
candidates.push(LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx,
|
||||
|
||||
@@ -2070,6 +2070,53 @@ mod test {
|
||||
assert!(meta.versions.iter().all(|version| !version.header.free_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bytes_count_toward_listing_limit_matches_fileinfo_visibility() {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut obj = FileInfo::new("object", 2, 2);
|
||||
obj.version_id = Some(Uuid::new_v4());
|
||||
obj.mod_time = Some(OffsetDateTime::now_utc());
|
||||
meta.add_version(obj.clone()).expect("add object version");
|
||||
let obj_bytes = meta.marshal_msg().expect("marshal object");
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(&obj_bytes));
|
||||
|
||||
let dm = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mark_deleted: true,
|
||||
mod_time: obj.mod_time.map(|t| t + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
};
|
||||
meta.delete_version(&dm).expect("add delete marker");
|
||||
let dm_bytes = meta.marshal_msg().expect("marshal delete marker");
|
||||
assert!(!FileMeta::bytes_count_toward_listing_limit(&dm_bytes));
|
||||
|
||||
let (_, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
|
||||
source.version_id = Some(Uuid::new_v4());
|
||||
let mut transitioned = FileMeta::new();
|
||||
transitioned.add_version(source.clone()).expect("seed transitioned version");
|
||||
let mut del_req = FileInfo {
|
||||
name: "object".to_string(),
|
||||
version_id: source.version_id,
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
del_req.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
transitioned
|
||||
.delete_version(&del_req)
|
||||
.expect("delete transitioned version leaving free version");
|
||||
assert!(!FileMeta::bytes_count_toward_listing_limit(
|
||||
&transitioned.marshal_msg().expect("marshal free-only")
|
||||
));
|
||||
|
||||
transitioned.add_version(obj).expect("add live object version");
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(
|
||||
&transitioned.marshal_msg().expect("marshal free + live object")
|
||||
));
|
||||
assert!(FileMeta::bytes_count_toward_listing_limit(b"corrupt-xl-meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_filemata_uses_canonical_equal_time_order() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
|
||||
|
||||
@@ -324,6 +324,63 @@ impl FileMeta {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn buffers_match_strict(self_buf: &[u8], other_buf: &[u8]) -> bool {
|
||||
let Ok((s_meta, _)) = Self::parse_validated_xl_envelope(self_buf) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_meta, _)) = Self::parse_validated_xl_envelope(other_buf) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if s_meta.is_empty() || o_meta.is_empty() {
|
||||
return s_meta.is_empty() && o_meta.is_empty();
|
||||
}
|
||||
|
||||
let Ok((s_len, s_header_ver, _s_meta_ver, s_meta)) = Self::decode_validated_xl_headers(s_meta) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_len, o_header_ver, _o_meta_ver, o_meta)) = Self::decode_validated_xl_headers(o_meta) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if s_len != o_len {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut s_cur: Cursor<&[u8]> = Cursor::new(s_meta);
|
||||
let mut o_cur: Cursor<&[u8]> = Cursor::new(o_meta);
|
||||
for _ in 0..s_len {
|
||||
let Ok((s_header_raw, _)) = Self::read_next_raw_slices(s_meta, &mut s_cur) else {
|
||||
return false;
|
||||
};
|
||||
let Ok((o_header_raw, _)) = Self::read_next_raw_slices(o_meta, &mut o_cur) else {
|
||||
return false;
|
||||
};
|
||||
if s_header_ver == XL_HEADER_VERSION
|
||||
&& o_header_ver == XL_HEADER_VERSION
|
||||
&& s_header_raw.len() == 38
|
||||
&& o_header_raw.len() == 38
|
||||
&& s_header_raw == o_header_raw
|
||||
&& parse_canonical_v3_header_fields(s_header_raw).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut s_header = FileMetaVersionHeader::default();
|
||||
if s_header.unmarshal_v(s_header_ver, s_header_raw).is_err() {
|
||||
return false;
|
||||
}
|
||||
let mut o_header = FileMetaVersionHeader::default();
|
||||
if o_header.unmarshal_v(o_header_ver, o_header_raw).is_err() {
|
||||
return false;
|
||||
}
|
||||
if !Self::headers_match_strict(&s_header, &o_header) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn header_matches_canonical_v3_raw_strict(s_header: &FileMetaVersionHeader, buf: &[u8]) -> Option<bool> {
|
||||
if buf.len() != 38 {
|
||||
@@ -625,6 +682,49 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bytes_count_toward_listing_limit(buf: &[u8]) -> bool {
|
||||
let Ok((meta, _inline_buf)) = Self::parse_validated_xl_envelope(buf) else {
|
||||
return true;
|
||||
};
|
||||
if meta.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Ok((versions_len, header_ver, _meta_ver, meta)) = Self::decode_validated_xl_headers(meta) else {
|
||||
return true;
|
||||
};
|
||||
if versions_len == 0 {
|
||||
return false;
|
||||
}
|
||||
let mut cur = Cursor::new(meta);
|
||||
let mut selected: Option<(FileMetaVersionHeader, &[u8])> = None;
|
||||
let mut any_non_hidden = false;
|
||||
for _ in 0..versions_len {
|
||||
let Ok((header, ver_meta)) = Self::read_next_shallow_slice(meta, &mut cur, header_ver) else {
|
||||
return true;
|
||||
};
|
||||
if header.version_type != VersionType::Delete && !header.free_version() {
|
||||
any_non_hidden = true;
|
||||
}
|
||||
if selected.is_some() {
|
||||
continue;
|
||||
}
|
||||
if header.free_version() && header.version_id.is_some_and(|id| !id.is_nil()) {
|
||||
continue;
|
||||
}
|
||||
selected = Some((header, ver_meta));
|
||||
}
|
||||
let Some((header, ver_meta)) = selected else {
|
||||
return false;
|
||||
};
|
||||
if !header.free_version() && header.version_type != VersionType::Delete {
|
||||
return true;
|
||||
}
|
||||
if !any_non_hidden {
|
||||
return false;
|
||||
}
|
||||
FileMetaVersion::decode_fileinfo_from_slice(ver_meta, "", "", true, false).is_err()
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "FileMeta")]
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
let mut wr =
|
||||
|
||||
@@ -352,6 +352,22 @@ impl MetaCacheEntry {
|
||||
(prefer, true)
|
||||
}
|
||||
|
||||
pub fn matches_strict_borrowed(&mut self, other: &MetaCacheEntry) -> bool {
|
||||
if self.name != other.name {
|
||||
return false;
|
||||
}
|
||||
|
||||
if other.is_dir() || self.is_dir() {
|
||||
return other.is_dir() == self.is_dir();
|
||||
}
|
||||
|
||||
if self.cached.is_none() && other.cached.is_none() {
|
||||
return FileMeta::buffers_match_strict(&self.metadata, &other.metadata);
|
||||
}
|
||||
|
||||
self.matches_strict(other)
|
||||
}
|
||||
|
||||
pub fn matches_strict(&mut self, other: &MetaCacheEntry) -> bool {
|
||||
if self.name != other.name {
|
||||
return false;
|
||||
@@ -909,6 +925,13 @@ impl MetaCacheEntriesSorted {
|
||||
|
||||
const METACACHE_STREAM_VERSION: u8 = 2;
|
||||
const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
|
||||
/// Upper bound on one incremental reservation while accumulating a single
|
||||
/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
|
||||
/// up to this bound so a legitimately large xl.meta does not pay one small
|
||||
/// read per chunk, while a corrupt length prefix still cannot force a
|
||||
/// reservation larger than the payload that has already arrived (see
|
||||
/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
|
||||
const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
|
||||
/// msgpack `false`: the entry-stream terminator, and the only byte a legacy
|
||||
/// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
|
||||
const METACACHE_TERMINATOR: u8 = 0xc2;
|
||||
@@ -989,6 +1012,17 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn write_obj_buffered(&mut self, obj: &MetaCacheEntry) -> Result<()> {
|
||||
self.init().await?;
|
||||
|
||||
Self::encode_obj(&mut self.buf, obj)?;
|
||||
if self.buf.len() >= METACACHE_READ_CHUNK_SIZE {
|
||||
self.flush().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn close(&mut self) -> Result<()> {
|
||||
self.close_with_limit_reached(false).await
|
||||
}
|
||||
@@ -1130,13 +1164,14 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
// A whole xl.meta can legitimately exceed the limit on its individual
|
||||
// msgpack values. Grow only as payload arrives, so a corrupt length
|
||||
// prefix cannot cause a speculative allocation of the advertised size.
|
||||
let mut chunk = METACACHE_READ_CHUNK_SIZE;
|
||||
while self.buf.len() < read_size {
|
||||
let start = self.buf.len();
|
||||
let extra = if lookahead {
|
||||
let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
|
||||
(read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
|
||||
(read_size - start).clamp(min_chunk, chunk)
|
||||
} else {
|
||||
(read_size - start).min(METACACHE_READ_CHUNK_SIZE)
|
||||
(read_size - start).min(chunk)
|
||||
};
|
||||
if let Err(e) = self.buf.try_reserve(extra) {
|
||||
let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
|
||||
@@ -1167,6 +1202,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
|
||||
}
|
||||
|
||||
self.pos = read_size;
|
||||
@@ -1282,6 +1318,85 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
|
||||
}
|
||||
|
||||
pub fn skip_current(&mut self) -> bool {
|
||||
if self.has_current && self.init && self.err.is_none() {
|
||||
self.has_current = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_peek_buffered(&mut self) -> Option<Result<Option<MetaCacheEntry>>> {
|
||||
if !self.init || self.err.is_some() || self.terminated {
|
||||
return None;
|
||||
}
|
||||
|
||||
let avail = &self.buf[self.pos..];
|
||||
let &first = avail.first()?;
|
||||
if first != 0xc3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let str_marker = *avail.get(1)?;
|
||||
let (str_hdr, str_len) = match str_marker {
|
||||
0xa0..=0xbf => (1usize, usize::from(str_marker & 0x1f)),
|
||||
0xd9 => (2usize, usize::from(*avail.get(2)?)),
|
||||
0xda => (3usize, usize::from(u16::from_be_bytes([*avail.get(2)?, *avail.get(3)?]))),
|
||||
0xdb => (
|
||||
5usize,
|
||||
usize::try_from(u32::from_be_bytes([*avail.get(2)?, *avail.get(3)?, *avail.get(4)?, *avail.get(5)?])).ok()?,
|
||||
),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let bin_offset = 1usize.checked_add(str_hdr)?.checked_add(str_len)?;
|
||||
let bin_marker = *avail.get(bin_offset)?;
|
||||
let (bin_hdr, bin_len) = match bin_marker {
|
||||
0xc4 => (2usize, usize::from(*avail.get(bin_offset + 1)?)),
|
||||
0xc5 => (
|
||||
3usize,
|
||||
usize::from(u16::from_be_bytes([*avail.get(bin_offset + 1)?, *avail.get(bin_offset + 2)?])),
|
||||
),
|
||||
0xc6 => (
|
||||
5usize,
|
||||
usize::try_from(u32::from_be_bytes([
|
||||
*avail.get(bin_offset + 1)?,
|
||||
*avail.get(bin_offset + 2)?,
|
||||
*avail.get(bin_offset + 3)?,
|
||||
*avail.get(bin_offset + 4)?,
|
||||
]))
|
||||
.ok()?,
|
||||
),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let total_len = bin_offset.checked_add(bin_hdr)?.checked_add(bin_len)?;
|
||||
if avail.len() < total_len {
|
||||
return None;
|
||||
}
|
||||
|
||||
let Ok(decoded) = from_utf8(&avail[1 + str_hdr..bin_offset]) else {
|
||||
return None;
|
||||
};
|
||||
let name = decoded.to_owned();
|
||||
let metadata = if bin_len == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
avail[bin_offset + bin_hdr..total_len].to_vec()
|
||||
};
|
||||
|
||||
self.pos += total_len;
|
||||
self.reset();
|
||||
self.has_current = true;
|
||||
Some(Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
metadata,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn skip(&mut self, size: usize) -> Result<()> {
|
||||
self.check_init().await?;
|
||||
|
||||
@@ -2049,6 +2164,76 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
|
||||
/// decode completely even when the aggregate stream exceeds the per-value
|
||||
/// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
|
||||
/// element.
|
||||
#[tokio::test]
|
||||
async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
|
||||
// ~2.9 KiB of metadata per entry matches the per-entry cost observed in
|
||||
// the #9120 production report; 6,000 entries push the stream past 16 MiB.
|
||||
const ENTRIES: usize = 6_000;
|
||||
let metadata = vec![0xab; 2_880];
|
||||
let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
|
||||
.map(|i| MetaCacheEntry {
|
||||
name: format!("bigdir/object-{i:05}"),
|
||||
metadata: metadata.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
writer.write(&entries).await.expect("wide listing must encode");
|
||||
writer.close().await.expect("stream must close");
|
||||
let bytes = output.into_inner();
|
||||
assert!(
|
||||
bytes.len() > 16 << 20,
|
||||
"fixture must exceed the per-value limit, got {} bytes",
|
||||
bytes.len()
|
||||
);
|
||||
let mut reader = MetacacheReader::new(Cursor::new(bytes));
|
||||
let decoded = reader.read_all().await.expect("wide listing must decode");
|
||||
assert_eq!(decoded, entries);
|
||||
}
|
||||
|
||||
/// The #9120 failure shape seen in production: one entry whose metadata is
|
||||
/// an xl.meta that grew past 16 MiB with its version history, listed among
|
||||
/// ordinary neighbors and skipped past without losing the framing.
|
||||
#[tokio::test]
|
||||
async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
|
||||
let small = |i: u8| MetaCacheEntry {
|
||||
name: format!("bigdir/object-{i}"),
|
||||
metadata: vec![i; 64],
|
||||
..Default::default()
|
||||
};
|
||||
let oversized = MetaCacheEntry {
|
||||
name: "bigdir/huge-versions".to_string(),
|
||||
metadata: vec![0xcd; (16 << 20) + 12_345],
|
||||
..Default::default()
|
||||
};
|
||||
let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
writer.write(&entries).await.expect("mixed listing must encode");
|
||||
writer.close().await.expect("stream must close");
|
||||
|
||||
let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
|
||||
assert_eq!(
|
||||
reader.peek().await.expect("first entry").expect("present"),
|
||||
entries[0],
|
||||
"an ordinary entry before the oversized one must decode"
|
||||
);
|
||||
reader.skip(1).await.expect("skip the first entry");
|
||||
assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
|
||||
assert_eq!(
|
||||
reader.peek().await.expect("oversized entry").expect("present"),
|
||||
oversized,
|
||||
"an entry past the per-value limit must round-trip whole"
|
||||
);
|
||||
reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
|
||||
assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
|
||||
let version_id = Uuid::new_v4();
|
||||
@@ -3669,6 +3854,74 @@ mod tests {
|
||||
assert!(corrupt_crc_entry.is_latest_delete_marker());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
|
||||
let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
|
||||
.expect("load realistic xl.meta");
|
||||
let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
|
||||
let mut free_meta_sys = HashMap::new();
|
||||
rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
|
||||
let free = FileMetaShallowVersion::try_from(FileMetaVersion {
|
||||
version_type: VersionType::Delete,
|
||||
delete_marker: Some(MetaDeleteMarker {
|
||||
version_id: Some(Uuid::from_u128(42)),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
|
||||
meta_sys: free_meta_sys,
|
||||
}),
|
||||
write_version: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("encode cleanup record");
|
||||
fm.versions.insert(0, free);
|
||||
let entry = MetaCacheEntry {
|
||||
name: "object".to_string(),
|
||||
metadata: fm.marshal_msg().expect("encode metadata"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let scanned = fm
|
||||
.get_file_info_versions("bucket", "object", false)
|
||||
.expect("scanner version decode");
|
||||
assert_eq!(scanned.free_versions.len(), 1);
|
||||
assert_eq!(scanned.versions.len(), 1);
|
||||
assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
|
||||
assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
|
||||
let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
|
||||
assert_eq!(listed.version_id, object.version_id);
|
||||
assert_eq!(listed.version_id, scanned.versions[0].version_id);
|
||||
assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
|
||||
assert_eq!(listed.size, scanned.versions[0].size);
|
||||
assert!(!listed.deleted);
|
||||
|
||||
let marker_id = Uuid::from_u128(43);
|
||||
fm.add_version(FileInfo {
|
||||
volume: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(marker_id),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("add current delete marker");
|
||||
let mut marker_entry = MetaCacheEntry {
|
||||
name: "object".to_string(),
|
||||
metadata: fm.marshal_msg().expect("encode marker metadata"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let scanned = fm
|
||||
.get_file_info_versions("bucket", "object", false)
|
||||
.expect("scan marker stack");
|
||||
assert_eq!(scanned.versions.len(), 2);
|
||||
assert_eq!(scanned.versions[0].version_id, Some(marker_id));
|
||||
assert!(scanned.versions[0].deleted);
|
||||
assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
|
||||
assert!(marker_entry.is_latest_delete_marker());
|
||||
let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
|
||||
assert_eq!(listed.version_id, scanned.versions[0].version_id);
|
||||
assert!(listed.deleted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
|
||||
let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
|
||||
@@ -4257,4 +4510,95 @@ mod tests {
|
||||
.expect("resolve mixed empty");
|
||||
assert_eq!(res_mixed_empty.cached.expect("cached").versions.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_strict_borrowed_matches_shallow_headers_and_rejects_corrupt_or_divergent_buffers() {
|
||||
let base_bytes = create_real_xlmeta().expect("create real xlmeta");
|
||||
let mut peer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
|
||||
// Mutate inner version payload (ver.meta) without changing shallow header.
|
||||
let mut ver = FileMetaVersion::try_from(peer_fm.versions[1].meta.as_slice()).expect("decode version");
|
||||
ver.object.as_mut().expect("object version").erasure_index += 1;
|
||||
peer_fm.versions[1].meta = ver.marshal_msg().expect("marshal peer version");
|
||||
let peer_bytes = peer_fm.marshal_msg().expect("marshal peer xlmeta");
|
||||
assert_ne!(base_bytes, peer_bytes, "peer shard has distinct erasure index payload");
|
||||
|
||||
let mut e1 = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: base_bytes.clone(),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let e1_clone = e1.clone();
|
||||
let e2 = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: peer_bytes,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
assert!(e1.matches_strict_borrowed(&e2));
|
||||
assert!(e1.matches_strict_borrowed(&e1_clone));
|
||||
|
||||
// Different shallow header must not match.
|
||||
let mut newer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
|
||||
newer_fm.versions[0].header.version_id = Some(Uuid::new_v4());
|
||||
let newer_entry = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: newer_fm.marshal_msg().expect("marshal newer xlmeta"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
assert!(!e1.matches_strict_borrowed(&newer_entry));
|
||||
|
||||
// Identical corrupt buffers must return false, never true via byte-equality shortcut.
|
||||
let mut corrupt = MetaCacheEntry {
|
||||
name: "obj-a".to_owned(),
|
||||
metadata: b"XL2 corrupt-payload".to_vec(),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let corrupt_peer = corrupt.clone();
|
||||
assert!(!corrupt.matches_strict_borrowed(&corrupt_peer));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_peek_buffered_and_skip_current_match_async_peek_and_skip() {
|
||||
let entries: Vec<_> = (0..5)
|
||||
.map(|i| MetaCacheEntry {
|
||||
name: format!("buffered-key-{i}"),
|
||||
metadata: vec![i as u8; 16],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})
|
||||
.collect();
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut buf);
|
||||
writer.write(&entries).await.expect("write entries");
|
||||
writer.close_with_limit_reached(false).await.expect("close stream");
|
||||
|
||||
let mut reader = MetacacheReader::new(Cursor::new(buf.into_inner()));
|
||||
assert!(reader.try_peek_buffered().is_none(), "uninitialized reader must defer to async peek");
|
||||
assert!(!reader.skip_current(), "no current entry before first peek");
|
||||
|
||||
let first = reader.peek().await.expect("first peek").expect("entry 0");
|
||||
assert_eq!(first, entries[0]);
|
||||
assert!(reader.skip_current(), "skip_current must consume the peeked entry");
|
||||
assert!(!reader.skip_current(), "second skip_current without peek must return false");
|
||||
|
||||
for expected in &entries[1..] {
|
||||
let next = reader
|
||||
.try_peek_buffered()
|
||||
.expect("remaining entries are already buffered")
|
||||
.expect("valid entry")
|
||||
.expect("present entry");
|
||||
assert_eq!(&next, expected);
|
||||
assert!(reader.skip_current());
|
||||
}
|
||||
|
||||
assert!(
|
||||
reader.try_peek_buffered().is_none(),
|
||||
"stream terminator must defer to async peek for trailer handling"
|
||||
);
|
||||
assert!(reader.peek().await.expect("eof peek").is_none());
|
||||
assert_eq!(reader.limit_reached(), Some(false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,6 +420,7 @@ async fn real_storage_grace_round_trip() {
|
||||
&HealOpts {
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
recreate: true,
|
||||
remove: true,
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
|
||||
@@ -810,7 +810,14 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
|
||||
fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
|
||||
active_heals
|
||||
.iter()
|
||||
.find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
|
||||
.find(|(_, task)| {
|
||||
PriorityHealQueue::make_dedup_key_for_owner(
|
||||
&task.heal_type,
|
||||
&task.options,
|
||||
task.source,
|
||||
task.expected_mrf_bucket_incarnation_id,
|
||||
) == key
|
||||
})
|
||||
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
||||
}
|
||||
|
||||
|
||||
@@ -611,7 +611,104 @@ impl PriorityHealQueue {
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
|
||||
Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
|
||||
Self::make_dedup_key_for_owner(
|
||||
&request.heal_type,
|
||||
&request.options,
|
||||
request.source,
|
||||
request.expected_mrf_bucket_incarnation_id,
|
||||
)
|
||||
}
|
||||
|
||||
/// Bound MRF work can only merge with work for the same bucket lifetime.
|
||||
/// Length-prefix user-controlled components so a key cannot forge another
|
||||
/// version, scope, or incarnation through delimiter characters.
|
||||
pub(super) fn make_dedup_key_for_owner(
|
||||
heal_type: &HealType,
|
||||
options: &HealOptions,
|
||||
source: HealRequestSource,
|
||||
incarnation: Option<Uuid>,
|
||||
) -> String {
|
||||
if source == HealRequestSource::Mrf
|
||||
&& let Some(incarnation) = incarnation
|
||||
{
|
||||
use std::fmt::Write as _;
|
||||
|
||||
let (pool, set) = heal_scope_indices(heal_type, options);
|
||||
let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
|
||||
let mut part = |value: &str| {
|
||||
// Formatting into a String is infallible.
|
||||
let _ = write!(key, "{}:{value}", value.len());
|
||||
};
|
||||
match heal_type {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
part("object");
|
||||
part(bucket);
|
||||
part(object);
|
||||
match version_id {
|
||||
None => part("none"),
|
||||
Some(version_id) => {
|
||||
part("some");
|
||||
part(version_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
part("metadata");
|
||||
part(bucket);
|
||||
part(object);
|
||||
}
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
part("ecdecode");
|
||||
part(bucket);
|
||||
part(object);
|
||||
match version_id {
|
||||
None => part("none"),
|
||||
Some(version_id) => {
|
||||
part("some");
|
||||
part(version_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealType::DeleteMarkerPurge {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
purge,
|
||||
} => {
|
||||
part("delete-marker-purge");
|
||||
part(bucket);
|
||||
part(object);
|
||||
part(version_id);
|
||||
part(&purge.bucket_incarnation_id.to_string());
|
||||
part(&purge.marker_incarnation_id.to_string());
|
||||
part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
part("bucket");
|
||||
part(bucket);
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
part("prefix");
|
||||
part(bucket);
|
||||
part(prefix);
|
||||
}
|
||||
// Cluster and erasure-set work are not bucket-bound; preserve
|
||||
// their existing keys, including contains_erasure_set lookups.
|
||||
HealType::Cluster | HealType::ErasureSet { .. } => {
|
||||
return Self::make_dedup_key_for_scope(heal_type, options);
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
Self::make_dedup_key_for_scope(heal_type, options)
|
||||
}
|
||||
|
||||
pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {
|
||||
|
||||
@@ -117,6 +117,453 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
|
||||
let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(incarnation);
|
||||
request.options.pool_index = Some(0);
|
||||
request.options.set_index = Some(0);
|
||||
request
|
||||
}
|
||||
|
||||
async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
|
||||
let old_incarnation = Uuid::new_v4();
|
||||
let new_incarnation = Uuid::new_v4();
|
||||
let old_request = incarnation_bound_mrf_fallback(old_incarnation);
|
||||
let old_id = old_request.id.clone();
|
||||
match existing_state {
|
||||
"queued" => {
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(old_request)
|
||||
.await
|
||||
.expect("old fallback must be admitted");
|
||||
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
|
||||
}
|
||||
"active" => {
|
||||
manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
|
||||
}
|
||||
"retrying" => {
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
old_id.clone(),
|
||||
RetryingHeal {
|
||||
request: old_request,
|
||||
error: "retry pending".to_owned(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
other => panic!("unknown fixture state: {other}"),
|
||||
}
|
||||
|
||||
let new_request = incarnation_bound_mrf_fallback(new_incarnation);
|
||||
let new_id = new_request.id.clone();
|
||||
assert_ne!(new_id, old_id, "the fallback must use a new request ID");
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(new_request)
|
||||
.await
|
||||
.expect("new incarnation admission must be decided");
|
||||
assert_eq!(
|
||||
receipt.result,
|
||||
HealAdmissionResult::Accepted,
|
||||
"a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
|
||||
);
|
||||
assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
|
||||
let queue = manager.heal_queue.lock().await;
|
||||
assert!(
|
||||
queue
|
||||
.requests()
|
||||
.any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
|
||||
);
|
||||
if existing_state == "queued" {
|
||||
assert!(
|
||||
queue
|
||||
.requests()
|
||||
.any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("active").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
|
||||
}
|
||||
|
||||
fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
|
||||
*id = version_id.map(str::to_owned);
|
||||
} else {
|
||||
unreachable!("fallback fixture must be an object request");
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
|
||||
for version in [Uuid::nil(), Uuid::new_v4()] {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let version_id = version.to_string();
|
||||
let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
|
||||
let original_id = original.id.clone();
|
||||
let accepted = manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first queued fallback");
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
let merged = manager
|
||||
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
|
||||
.await
|
||||
.expect("duplicate queued fallback");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, original_id);
|
||||
assert_eq!(manager.heal_queue.lock().await.len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let original = incarnation_bound_mrf_fallback(incarnation);
|
||||
let original_id = original.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let same = manager
|
||||
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
|
||||
.await
|
||||
.expect("same-incarnation fallback");
|
||||
assert_eq!(same.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(same.task_id, original_id);
|
||||
|
||||
let mut versioned = incarnation_bound_mrf_fallback(incarnation);
|
||||
versioned.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
};
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(versioned)
|
||||
.await
|
||||
.expect("different version")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
|
||||
other_scope.options.pool_index = Some(1);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(other_scope)
|
||||
.await
|
||||
.expect("different pool")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
same_id_other_incarnation.id = original_id;
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(same_id_other_incarnation)
|
||||
.await
|
||||
.expect("same-ID conflict must be decided")
|
||||
.result,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let bound = incarnation_bound_mrf_fallback(incarnation);
|
||||
let mut unbound = bound.clone();
|
||||
unbound.expected_mrf_bucket_incarnation_id = None;
|
||||
let mut non_mrf = bound.clone();
|
||||
non_mrf.source = HealRequestSource::ReadRepair;
|
||||
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key(&non_mrf),
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
|
||||
"an unrelated source retains its previous scope key even if the MRF-only field is set"
|
||||
);
|
||||
assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
|
||||
|
||||
let mut left = bound.clone();
|
||||
left.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "part:a".to_owned(),
|
||||
version_id: Some("b".to_owned()),
|
||||
};
|
||||
let mut right = bound.clone();
|
||||
right.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "part".to_owned(),
|
||||
version_id: Some("a:b".to_owned()),
|
||||
};
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
|
||||
"the legacy delimiter-only key demonstrates the collision"
|
||||
);
|
||||
assert_ne!(
|
||||
PriorityHealQueue::make_dedup_key(&left),
|
||||
PriorityHealQueue::make_dedup_key(&right),
|
||||
"length-prefixed object and version components must not alias"
|
||||
);
|
||||
let mut forged = unbound.clone();
|
||||
forged.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: PriorityHealQueue::make_dedup_key(&bound),
|
||||
version_id: None,
|
||||
};
|
||||
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
for decode in [false, true] {
|
||||
let make_request = |object: &str, version_id: Option<&str>| {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
let bucket = "mrf-recreated-bucket".to_owned();
|
||||
let object = object.to_owned();
|
||||
let version_id = version_id.map(str::to_owned);
|
||||
request.heal_type = if decode {
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
} else {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
};
|
||||
request
|
||||
};
|
||||
let absent = make_request("object", None);
|
||||
let empty = make_request("object", Some(""));
|
||||
let forged_object = make_request("object4:some0:", None);
|
||||
let forged_version = make_request("object", Some("4:none"));
|
||||
let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
assert!(keys.iter().skip(index + 1).all(|other| key != other));
|
||||
}
|
||||
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
|
||||
assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
|
||||
assert_eq!(queue.len(), 2);
|
||||
|
||||
for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
|
||||
let mut legacy_absent = absent.clone();
|
||||
let mut legacy_empty = empty.clone();
|
||||
legacy_absent.source = source;
|
||||
legacy_empty.source = source;
|
||||
if source == HealRequestSource::Mrf {
|
||||
legacy_absent.expected_mrf_bucket_incarnation_id = None;
|
||||
legacy_empty.expected_mrf_bucket_incarnation_id = None;
|
||||
}
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key(&legacy_absent),
|
||||
PriorityHealQueue::make_dedup_key(&legacy_empty),
|
||||
"unbound and non-MRF work retain the legacy empty-version key"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
let old_id = old.id.clone();
|
||||
let old_key = PriorityHealQueue::make_dedup_key(&old);
|
||||
let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
let new_id = new.id.clone();
|
||||
let new_key = PriorityHealQueue::make_dedup_key(&new);
|
||||
let mut forced_old = old.clone();
|
||||
forced_old.id = Uuid::new_v4().to_string();
|
||||
forced_old.force_start = true;
|
||||
let forced_id = forced_old.id.clone();
|
||||
|
||||
assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
|
||||
queue
|
||||
.remove_request_id(&old_id)
|
||||
.expect("remove the old incarnation representative");
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
|
||||
assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
|
||||
assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
|
||||
assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
|
||||
assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
|
||||
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
|
||||
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(old)
|
||||
.await
|
||||
.expect("old fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let request = incarnation_bound_mrf_fallback(incarnation);
|
||||
let request_id = request.id.clone();
|
||||
let mut intent = MrfIntent {
|
||||
bucket: Arc::from("mrf-recreated-bucket"),
|
||||
object: Arc::from("object"),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
};
|
||||
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
|
||||
let target = MrfRepairNoticeTarget {
|
||||
bucket: Arc::clone(&intent.bucket),
|
||||
object: Arc::clone(&intent.object),
|
||||
version_id: intent.version_id,
|
||||
kind: intent.kind,
|
||||
scope: intent.scope,
|
||||
delete_marker_purge: None,
|
||||
lease: intent.lease,
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
|
||||
.await
|
||||
.expect("durable admission must be decided");
|
||||
assert_eq!(
|
||||
receipt.result,
|
||||
HealAdmissionResult::Accepted,
|
||||
"a durable new incarnation needs its own owner"
|
||||
);
|
||||
assert_eq!(receipt.task_id, request_id);
|
||||
assert!(
|
||||
manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.requests()
|
||||
.any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.try_in_flight_durable_mrf_anchors()
|
||||
.expect("stable owner snapshot")
|
||||
.contains(&anchor)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
|
||||
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
|
||||
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let object = format!("same-generation-{}", Uuid::new_v4());
|
||||
let make_request = || {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
request.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: object.clone(),
|
||||
version_id: None,
|
||||
};
|
||||
request
|
||||
};
|
||||
let old = make_request();
|
||||
let old_id = old.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(old)
|
||||
.await
|
||||
.expect("old fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut intent = MrfIntent {
|
||||
bucket: Arc::from("mrf-recreated-bucket"),
|
||||
object: Arc::from(object.as_str()),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
};
|
||||
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
|
||||
let target = MrfRepairNoticeTarget {
|
||||
bucket: Arc::clone(&intent.bucket),
|
||||
object: Arc::clone(&intent.object),
|
||||
version_id: intent.version_id,
|
||||
kind: intent.kind,
|
||||
scope: intent.scope,
|
||||
delete_marker_purge: None,
|
||||
lease: intent.lease,
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let merged = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
|
||||
.await
|
||||
.expect("durable same-incarnation admission");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, old_id);
|
||||
assert!(
|
||||
manager
|
||||
.try_in_flight_durable_mrf_anchors()
|
||||
.expect("stable owner snapshot")
|
||||
.contains(&anchor)
|
||||
);
|
||||
}
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
task_id: "fixture-token".to_string(),
|
||||
|
||||
@@ -480,8 +480,9 @@ impl HealTask {
|
||||
message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
|
||||
})?;
|
||||
if current_bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
|
||||
return Err(Error::StaleBucketIncarnation {
|
||||
bucket: bucket.to_owned(),
|
||||
expected: Some(bucket_incarnation_id),
|
||||
});
|
||||
}
|
||||
expected.bucket_incarnation_id = Some(bucket_incarnation_id);
|
||||
|
||||
@@ -3375,10 +3375,67 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
|
||||
|
||||
let result = task.heal_object("bucket", "object", None).await;
|
||||
|
||||
assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
|
||||
));
|
||||
assert!(storage.heal_object_calls.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
|
||||
for object in ["timed out", "network error", "ordinary"] {
|
||||
let original_incarnation = Uuid::new_v4();
|
||||
let recreated_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
|
||||
request.options.pool_index = Some(0);
|
||||
request.options.set_index = Some(1);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("a recreated bucket must reject the old MRF responsibility");
|
||||
assert!(
|
||||
!error.is_recoverable_heal(),
|
||||
"user-controlled object names must not make a stale incarnation retryable: {object}"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
&error,
|
||||
Error::StaleBucketIncarnation { bucket, expected }
|
||||
if bucket == "bucket" && *expected == Some(original_incarnation)
|
||||
),
|
||||
"the failure must retain the original bucket incarnation: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
|
||||
"stale responsibility must be classified independently of the object name"
|
||||
);
|
||||
assert!(
|
||||
storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
|
||||
"an old responsibility must not reach storage repair in the recreated bucket"
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
|
||||
assert!(
|
||||
outcome.objects.iter().all(|item| !matches!(
|
||||
&item.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)),
|
||||
"the stale task must not produce a positive repair proof"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
|
||||
@@ -157,7 +157,7 @@ async fn wait_for_object_copies(disks: &[PathBuf], bucket: &str, object: &str) {
|
||||
fn deep_heal_opts() -> HealOpts {
|
||||
HealOpts {
|
||||
recreate: true,
|
||||
remove: false,
|
||||
remove: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
}
|
||||
@@ -633,6 +633,7 @@ mod absence_receipt_regressions {
|
||||
version_id: Some(old.to_owned()),
|
||||
},
|
||||
HealOptions {
|
||||
remove_corrupted: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
@@ -823,6 +824,7 @@ mod absence_receipt_regressions {
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
remove_corrupted: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -810,7 +810,13 @@ fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
|
||||
restarted.start().await.expect("restart with retained failed batch");
|
||||
let restored = restarted.get_task_report(&token).await.expect("original token after restart");
|
||||
assert_eq!(restored.status, report.status);
|
||||
assert_eq!(restored.outcome, report.outcome);
|
||||
// Quorum and Deep observations are local and never persisted.
|
||||
let mut expected_outcome = report.outcome.as_deref().cloned().expect("mixed batch outcome");
|
||||
for object in &mut expected_outcome.objects {
|
||||
object.quorum_selected_data_dir = None;
|
||||
object.deep_check_observed = false;
|
||||
}
|
||||
assert_eq!(restored.outcome.as_deref(), Some(&expected_outcome));
|
||||
assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
|
||||
restarted.stop().await.expect("stop restarted mixed batch manager");
|
||||
});
|
||||
|
||||
@@ -16,9 +16,10 @@ use super::{
|
||||
transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
|
||||
};
|
||||
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
|
||||
use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
|
||||
use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
|
||||
use openidconnect::{
|
||||
AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
|
||||
AsyncHttpClient, ClaimsVerificationError, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
|
||||
ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
@@ -212,6 +213,35 @@ impl DiscoveredProviderMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
|
||||
token: &'a CoreIdToken,
|
||||
verifier: &CoreIdTokenVerifier<'_>,
|
||||
nonce_verifier: N,
|
||||
) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
|
||||
let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
|
||||
CoreJwsSigningAlgorithm::HmacSha256 => 32,
|
||||
CoreJwsSigningAlgorithm::HmacSha384 => 48,
|
||||
CoreJwsSigningAlgorithm::HmacSha512 => 64,
|
||||
_ => return token.claims(verifier, nonce_verifier),
|
||||
};
|
||||
// RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
|
||||
// openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
|
||||
let invalid_length = || {
|
||||
ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
|
||||
"invalid HMAC signature length".to_owned(),
|
||||
))
|
||||
};
|
||||
let compact = token.to_string();
|
||||
let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
|
||||
let signature = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(signature)
|
||||
.map_err(|_| invalid_length())?;
|
||||
if signature.len() != expected_length {
|
||||
return Err(invalid_length());
|
||||
}
|
||||
token.claims(verifier, nonce_verifier)
|
||||
}
|
||||
|
||||
// This adapter is used only for discovery/JWKS fetches, never token exchange.
|
||||
struct JwksAcceptClient<'a> {
|
||||
inner: &'a ReqwestHttpClient,
|
||||
|
||||
@@ -242,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let payload = serde_json::json!({
|
||||
"iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
|
||||
"iat": now, "exp": now + 300,
|
||||
"iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
|
||||
});
|
||||
|
||||
for (algorithm, name, signature_length) in [
|
||||
@@ -275,22 +275,28 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
|
||||
.expect("parse configured provider metadata");
|
||||
let verifier = metadata.verifier(&config);
|
||||
assert!(
|
||||
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
"valid {name} token must pass for workload={workload}"
|
||||
);
|
||||
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
|
||||
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
|
||||
|
||||
for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
|
||||
let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
|
||||
let malformed: CoreIdToken = format!("{signing_input}.{signature}")
|
||||
.parse()
|
||||
.expect("invalid signature length must still reach cryptographic verification");
|
||||
.expect("invalid signature length must remain parseable before verification");
|
||||
assert!(
|
||||
malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"{name} signature length {length} must return an error for workload={workload}"
|
||||
);
|
||||
assert!(
|
||||
verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
|
||||
"{name} signature length {length} must return an error with a nonce for workload={workload}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
"invalid inputs must not prevent later valid {name} verification"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::{
|
||||
OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
|
||||
SourcedOidcProviderConfig, load_oidc_config_snapshot,
|
||||
},
|
||||
provider::{ProviderRuntime, discover_provider_for_recovery},
|
||||
provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
|
||||
state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
|
||||
transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
|
||||
};
|
||||
@@ -535,7 +535,7 @@ impl OidcSys {
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
|
||||
let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
|
||||
if let Err(e) = verified {
|
||||
let verification_error = e.to_string();
|
||||
warn!(
|
||||
@@ -583,8 +583,7 @@ impl OidcSys {
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
id_token
|
||||
.claims(&verifier, &Nonce::new(session.nonce.clone()))
|
||||
verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
|
||||
.map_err(|retry_err| {
|
||||
error!(
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
@@ -767,7 +766,7 @@ impl OidcSys {
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
state = self
|
||||
.provider_runtime
|
||||
.refresh_provider_state(&provider_id, &config)
|
||||
@@ -780,8 +779,7 @@ impl OidcSys {
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
id_token
|
||||
.claims(&verifier, |_: Option<&Nonce>| Ok(()))
|
||||
verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
|
||||
.map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
|
||||
}
|
||||
|
||||
|
||||
@@ -1411,6 +1411,28 @@ pub fn record_peer_reachable(addr: &str) {
|
||||
publish_offline_gauge(&peers);
|
||||
}
|
||||
|
||||
/// Record a successful interaction with a cluster peer, but only when the peer is currently
|
||||
/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
|
||||
/// and lock-path successes that do not go through a fresh dial.
|
||||
///
|
||||
/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
|
||||
/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
|
||||
/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
|
||||
pub fn record_peer_reachable_if_offline(addr: &str) {
|
||||
let key = normalize_peer_key(addr);
|
||||
// Fast path: read lock to check whether the peer is currently offline.
|
||||
{
|
||||
let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match peers.get(key) {
|
||||
None => return,
|
||||
Some(entry) if entry.online => return,
|
||||
Some(_) => {} // offline: fall through to the write path
|
||||
}
|
||||
}
|
||||
// Slow path: the peer is offline, so flip it back online and republish the gauge.
|
||||
record_peer_reachable(addr);
|
||||
}
|
||||
|
||||
/// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
|
||||
/// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
|
||||
pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
|
||||
@@ -2188,6 +2210,34 @@ mod tests {
|
||||
assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
|
||||
let addr = "http://cluster-peer-recover-if-offline-test:9000";
|
||||
|
||||
// Mark the peer offline.
|
||||
record_peer_unreachable(addr, 1);
|
||||
assert!(cluster_peer_is_offline(addr));
|
||||
|
||||
// A successful cached-channel RPC (no fresh dial) clears the offline state.
|
||||
record_peer_reachable_if_offline(addr);
|
||||
assert!(!cluster_peer_is_offline(addr));
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_peer_reachable_if_offline_no_op_for_online_peer() {
|
||||
let addr = "http://cluster-peer-recover-noop-test:9000";
|
||||
|
||||
record_peer_reachable(addr);
|
||||
// Should not panic or change state for an already-online peer.
|
||||
record_peer_reachable_if_offline(addr);
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
|
||||
// Should also be a no-op for unknown peers.
|
||||
record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
|
||||
assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_should_bypass_is_self_healing() {
|
||||
let addr = "http://cluster-peer-bypass-selfheal-test:9000";
|
||||
|
||||
@@ -30,6 +30,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-tls-runtime?/fips"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -110,6 +110,9 @@ where
|
||||
ftps_required = self.config.ftps_required,
|
||||
"FTPS server starting"
|
||||
);
|
||||
if self.config.tls_enabled && self.config.cert_dir.is_some() {
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
|
||||
}
|
||||
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
|
||||
|
||||
let storage_clone = self.storage.clone();
|
||||
@@ -191,8 +194,6 @@ where
|
||||
);
|
||||
|
||||
// Build ServerConfig with SNI support
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let server_config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_cert_resolver(resolver);
|
||||
|
||||
@@ -43,3 +43,70 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
|
||||
|
||||
#[cfg(feature = "sftp")]
|
||||
pub use sftp::{SftpConfig, SftpInitError, SftpServer};
|
||||
|
||||
#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
|
||||
mod fips_tests {
|
||||
use crate::common::dummy_storage::DummyBackend;
|
||||
use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
|
||||
|
||||
#[test]
|
||||
fn protocols_reject_incompatible_host_provider() {
|
||||
const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
|
||||
if std::env::var_os(CHILD).is_some() {
|
||||
let mut provider = rustfs_tls_runtime::default_crypto_provider();
|
||||
provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
|
||||
provider.install_default().expect("install incompatible host provider");
|
||||
let cert_dir = tempfile::tempdir().expect("certificate directory");
|
||||
let cert_dir = cert_dir.path().to_string_lossy().into_owned();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
runtime.block_on(async {
|
||||
let ftps_config = FtpsConfig {
|
||||
cert_dir: Some(cert_dir.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
|
||||
.await
|
||||
.expect("FTPS config");
|
||||
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
|
||||
let error = ftps
|
||||
.start(receiver)
|
||||
.await
|
||||
.expect_err("FTPS must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
|
||||
let webdav_config = WebDavConfig {
|
||||
bind_addr: ([127, 0, 0, 1], 0).into(),
|
||||
cert_dir: Some(cert_dir),
|
||||
..Default::default()
|
||||
};
|
||||
let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
|
||||
.await
|
||||
.expect("WebDAV config");
|
||||
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
|
||||
let error = webdav
|
||||
.start(receiver)
|
||||
.await
|
||||
.expect_err("WebDAV must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
});
|
||||
return;
|
||||
}
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.arg("protocols_reject_incompatible_host_provider")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD, "1")
|
||||
.env("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||
.env("no_proxy", "localhost,127.0.0.1,::1")
|
||||
.output()
|
||||
.expect("start isolated protocol TLS test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +118,10 @@ where
|
||||
"WebDAV server starting"
|
||||
);
|
||||
|
||||
if self.config.tls_enabled && self.config.cert_dir.is_some() {
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
|
||||
}
|
||||
|
||||
let listener = TcpListener::bind(self.config.bind_addr).await?;
|
||||
self.serve(listener, shutdown_rx).await
|
||||
}
|
||||
@@ -155,8 +159,6 @@ where
|
||||
reload_shutdown_rx.clone(),
|
||||
);
|
||||
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
|
||||
|
||||
Some(TlsAcceptor::from(Arc::new(server_config)))
|
||||
|
||||
@@ -1081,7 +1081,7 @@ impl ChecksumHasher for Sha512Hasher {
|
||||
/// MD5 hasher for the ADDITIONAL checksum (x-amz-checksum-md5). Separate from the
|
||||
/// legacy Content-MD5 / ETag machinery — this only serves the flexible-checksum path.
|
||||
pub struct Md5Hasher {
|
||||
hasher: md5::Md5,
|
||||
hasher: rustfs_utils::FastMd5,
|
||||
}
|
||||
|
||||
impl Default for Md5Hasher {
|
||||
@@ -1092,14 +1092,14 @@ impl Default for Md5Hasher {
|
||||
|
||||
impl Md5Hasher {
|
||||
pub fn new() -> Self {
|
||||
use md5::Digest as _;
|
||||
Self { hasher: md5::Md5::new() }
|
||||
Self {
|
||||
hasher: rustfs_utils::FastMd5::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Md5Hasher {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
use md5::Digest as _;
|
||||
self.hasher.update(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
@@ -1111,13 +1111,11 @@ impl Write for Md5Hasher {
|
||||
|
||||
impl ChecksumHasher for Md5Hasher {
|
||||
fn finalize(&mut self) -> Vec<u8> {
|
||||
use md5::Digest as _;
|
||||
self.hasher.clone().finalize().to_vec()
|
||||
self.hasher.finalize_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
use md5::Digest as _;
|
||||
self.hasher = md5::Md5::new();
|
||||
self.hasher.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+102
-28
@@ -15,8 +15,8 @@
|
||||
use crate::compress_index::{Index, TryGetIndex};
|
||||
use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use md5::{Digest, Md5};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::FastMd5;
|
||||
use sha2::Sha256;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
@@ -27,11 +27,12 @@ pin_project! {
|
||||
pub struct EtagReader<R> {
|
||||
#[pin]
|
||||
pub inner: R,
|
||||
pub md5: Md5,
|
||||
pub md5: FastMd5,
|
||||
pub finished: bool,
|
||||
pub checksum: Option<String>,
|
||||
resolved_etag: Option<String>,
|
||||
encrypted_etag_mac: Option<Hmac<Sha256>>,
|
||||
delegate_inner_etag: bool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,11 +40,12 @@ impl<R> EtagReader<R> {
|
||||
pub fn new(inner: R, checksum: Option<String>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
md5: Md5::new(),
|
||||
md5: FastMd5::new(),
|
||||
finished: false,
|
||||
checksum,
|
||||
resolved_etag: None,
|
||||
encrypted_etag_mac: None,
|
||||
delegate_inner_etag: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +60,17 @@ impl<R> EtagReader<R> {
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
/// Reuse the inner reader's plaintext ETag when deriving an encrypted ETag,
|
||||
/// avoiding a redundant second MD5 pass over the same plaintext stream.
|
||||
pub fn with_inner_etag_delegation(mut self, delegate: bool) -> Self
|
||||
where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
self.delegate_inner_etag =
|
||||
delegate && self.encrypted_etag_mac.is_some() && self.checksum.is_none() && self.inner.is_etag_reader();
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the final md5 value (etag) as a hex string, only compute once.
|
||||
/// Can be called multiple times, always returns the same result after finished.
|
||||
pub fn get_etag(&mut self) -> String {
|
||||
@@ -65,8 +78,8 @@ impl<R> EtagReader<R> {
|
||||
return etag.clone();
|
||||
}
|
||||
|
||||
let etag = self.md5.clone().finalize().to_vec();
|
||||
let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let digest = self.md5.finalize_bytes();
|
||||
let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
self.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
}
|
||||
@@ -87,30 +100,32 @@ where
|
||||
if let Poll::Ready(Ok(())) = &poll {
|
||||
let filled = &buf.filled()[orig_filled..];
|
||||
if !filled.is_empty() {
|
||||
this.md5.update(filled);
|
||||
if !*this.delegate_inner_etag {
|
||||
this.md5.update(filled);
|
||||
}
|
||||
} else {
|
||||
// EOF
|
||||
*this.finished = true;
|
||||
let etag = if let Some(etag) = this.resolved_etag.as_ref() {
|
||||
etag.clone()
|
||||
} else {
|
||||
let etag = this.md5.clone().finalize().to_vec();
|
||||
let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
*this.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
};
|
||||
if let Some(checksum) = this.checksum {
|
||||
let etag = if let Some(etag) = this.resolved_etag.as_ref() {
|
||||
etag.clone()
|
||||
} else {
|
||||
let digest = this.md5.finalize_bytes();
|
||||
let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
*this.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
};
|
||||
|
||||
if let Some(checksum) = this.checksum
|
||||
&& *checksum != etag
|
||||
{
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
BadDigest {
|
||||
expected_md5: checksum.clone(),
|
||||
calculated_md5: etag,
|
||||
},
|
||||
)));
|
||||
if *checksum != etag {
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
BadDigest {
|
||||
expected_md5: checksum.clone(),
|
||||
calculated_md5: etag,
|
||||
},
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,16 +133,24 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> EtagResolvable for EtagReader<R> {
|
||||
impl<R> EtagResolvable for EtagReader<R>
|
||||
where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
true
|
||||
!self.delegate_inner_etag || self.inner.is_etag_reader()
|
||||
}
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
if let Some(mut mac) = self.encrypted_etag_mac.clone() {
|
||||
if !self.finished {
|
||||
return None;
|
||||
}
|
||||
mac.update(self.get_etag().as_bytes());
|
||||
let plaintext_etag = if self.delegate_inner_etag {
|
||||
self.inner.try_resolve_etag()?
|
||||
} else {
|
||||
self.get_etag()
|
||||
};
|
||||
mac.update(plaintext_etag.as_bytes());
|
||||
let protected = mac.finalize().into_bytes();
|
||||
return Some(hex_simd::encode_to_string(&protected[..16], hex_simd::AsciiCase::Lower));
|
||||
}
|
||||
@@ -167,6 +190,7 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use md5::{Digest, Md5};
|
||||
use rand::RngExt;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
@@ -340,4 +364,54 @@ mod tests {
|
||||
assert_eq!(digest.expected_md5, wrong_checksum);
|
||||
assert_eq!(digest.calculated_md5, calculated_md5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_etag_reader_with_inner_delegation_matches_standalone() {
|
||||
let data = b"delegated encrypted etag payload across multiple 64-byte MD5 blocks for verification";
|
||||
let key = [0x42; 32];
|
||||
let nonce = [0x24; 12];
|
||||
|
||||
let mut standalone =
|
||||
EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce).expect("standalone encrypted EtagReader");
|
||||
let mut out_standalone = Vec::new();
|
||||
standalone.read_to_end(&mut out_standalone).await.expect("read standalone");
|
||||
let expected_tag = standalone.try_resolve_etag().expect("standalone etag");
|
||||
|
||||
let inner =
|
||||
crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
|
||||
.expect("inner HashReader");
|
||||
assert!(inner.is_etag_reader(), "HashReader with diskable_md5=false must report is_etag_reader");
|
||||
let mut delegated = EtagReader::new_encrypted(inner, key, nonce)
|
||||
.expect("delegated encrypted EtagReader")
|
||||
.with_inner_etag_delegation(true);
|
||||
let mut out_delegated = Vec::new();
|
||||
delegated.read_to_end(&mut out_delegated).await.expect("read delegated");
|
||||
assert_eq!(out_delegated, data);
|
||||
assert_eq!(
|
||||
delegated.try_resolve_etag().as_deref(),
|
||||
Some(expected_tag.as_str()),
|
||||
"delegated encrypted EtagReader must produce identical HMAC ETag without re-hashing plaintext"
|
||||
);
|
||||
|
||||
let truncated = crate::LimitReader::new(
|
||||
crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
|
||||
.expect("inner HashReader for limit"),
|
||||
16,
|
||||
);
|
||||
assert!(
|
||||
!truncated.is_etag_reader(),
|
||||
"truncating LimitReader must not advertise inner ETag delegation"
|
||||
);
|
||||
|
||||
let mut non_etag_inner = EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce)
|
||||
.expect("encrypted EtagReader over Cursor")
|
||||
.with_inner_etag_delegation(true);
|
||||
let mut out_cursor = Vec::new();
|
||||
non_etag_inner.read_to_end(&mut out_cursor).await.expect("read cursor");
|
||||
assert_eq!(
|
||||
non_etag_inner.try_resolve_etag().as_deref(),
|
||||
Some(expected_tag.as_str()),
|
||||
"delegation request on non-EtagResolvable inner must keep outer MD5 hashing active"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -630,6 +630,10 @@ impl AsyncRead for HashReader {
|
||||
}
|
||||
|
||||
impl EtagResolvable for HashReader {
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
!self.diskable_md5 && self.inner.is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
if self.diskable_md5 {
|
||||
return None;
|
||||
|
||||
@@ -201,10 +201,31 @@ where
|
||||
boxed_reader(WarpReader::new(reader))
|
||||
}
|
||||
|
||||
impl<T> EtagResolvable for std::io::Cursor<T> {}
|
||||
|
||||
impl EtagResolvable for &[u8] {}
|
||||
|
||||
impl<R> EtagResolvable for tokio::io::BufReader<R>
|
||||
where
|
||||
R: tokio::io::AsyncRead + EtagResolvable,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.get_ref().is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.get_mut().try_resolve_etag()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> EtagResolvable for Box<T>
|
||||
where
|
||||
T: EtagResolvable + ?Sized,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.as_ref().is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.as_mut().try_resolve_etag()
|
||||
}
|
||||
|
||||
@@ -25,6 +25,10 @@ keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-tls-runtime/fips"]
|
||||
|
||||
[dependencies]
|
||||
rustfs-checksums.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
|
||||
@@ -81,7 +81,7 @@ use time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::{debug, error, trace, warn};
|
||||
use tracing::{error, trace, warn};
|
||||
use url::{Url, form_urlencoded};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -448,16 +448,7 @@ impl TransitionClient {
|
||||
tier_type: &str,
|
||||
timeouts: TransitionClientTimeouts,
|
||||
) -> Result<TransitionClient, std::io::Error> {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
// No default provider is set yet; try to install aws-lc-rs.
|
||||
// `install_default` can only fail if another thread races us and installs a provider
|
||||
// between our check and this call, which is still safe to ignore.
|
||||
if rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() {
|
||||
debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install");
|
||||
}
|
||||
} else {
|
||||
debug!("rustls crypto provider already installed, skipping aws-lc-rs install");
|
||||
}
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(std::io::Error::other)?;
|
||||
|
||||
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
|
||||
let timeouts = timeouts.validate()?;
|
||||
@@ -1862,21 +1853,36 @@ mod tests {
|
||||
assert!(outcome.is_ok(), "TLS config creation should not panic");
|
||||
}
|
||||
|
||||
/// Installing the rustls crypto provider when one is already set must not panic or return
|
||||
/// an error that surfaces to callers (the race-safe `get_default` check guards the install).
|
||||
#[cfg(feature = "fips")]
|
||||
#[test]
|
||||
fn provider_install_is_idempotent() {
|
||||
// Install once (may already be set by another test in this binary — that's fine).
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
// A second install attempt on an already-set provider must not panic.
|
||||
let outcome = std::panic::catch_unwind(|| {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
}
|
||||
// If a default is already present, the branch above is simply skipped.
|
||||
});
|
||||
assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
|
||||
fn fips_transition_rejects_incompatible_host_provider() {
|
||||
const CHILD: &str = "RUSTFS_TEST_TRANSITION_FIPS_PROVIDER";
|
||||
if std::env::var_os(CHILD).is_some() {
|
||||
let mut provider = rustfs_tls_runtime::default_crypto_provider();
|
||||
provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
|
||||
provider.install_default().expect("install incompatible host provider");
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let error = runtime
|
||||
.block_on(super::TransitionClient::new("127.0.0.1:0", super::Options::default(), ""))
|
||||
.expect_err("transition client must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
return;
|
||||
}
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.arg("fips_transition_rejects_incompatible_host_provider")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("start isolated transition TLS test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -640,12 +640,12 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
|
||||
GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
|
||||
GetRange::Offset(start) => HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: *start as i64,
|
||||
start: i64::try_from(*start).unwrap_or(i64::MAX),
|
||||
end: -1,
|
||||
},
|
||||
GetRange::Suffix(length) => HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: *length as i64,
|
||||
start: i64::try_from(*length).unwrap_or(i64::MAX),
|
||||
end: -1,
|
||||
},
|
||||
}
|
||||
@@ -654,15 +654,15 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
|
||||
fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: range.start as i64,
|
||||
end: range.end.saturating_sub(1) as i64,
|
||||
start: i64::try_from(range.start).unwrap_or(i64::MAX),
|
||||
end: i64::try_from(range.end.saturating_sub(1)).unwrap_or(i64::MAX),
|
||||
}
|
||||
}
|
||||
|
||||
fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: start as i64,
|
||||
start: i64::try_from(start).unwrap_or(i64::MAX),
|
||||
end: -1,
|
||||
}
|
||||
}
|
||||
@@ -2476,6 +2476,48 @@ mod test {
|
||||
assert_eq!(range.end, 19);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_suffix_ranges() {
|
||||
// Normal suffix: last 100 bytes.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Suffix(100));
|
||||
assert!(range.is_suffix_length);
|
||||
assert_eq!(range.start, 100);
|
||||
assert_eq!(range.end, -1);
|
||||
|
||||
// Overflowing suffix: `u64::MAX` previously wrapped to `-1` via `as i64`
|
||||
// and was misread as "last 1 byte". It now clamps to `i64::MAX`, which
|
||||
// `get_length` treats as "the whole object".
|
||||
let range = http_range_spec_from_get_range(&GetRange::Suffix(u64::MAX));
|
||||
assert!(range.is_suffix_length);
|
||||
assert_eq!(range.start, i64::MAX);
|
||||
assert_eq!(range.end, -1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_offset_overflow() {
|
||||
// Normal offset.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Offset(42));
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!(range.start, 42);
|
||||
assert_eq!(range.end, -1);
|
||||
|
||||
// Overflowing offset clamps to `i64::MAX`, which is then rejected as
|
||||
// out-of-range (416) by `get_offset_length`.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Offset(u64::MAX));
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!(range.start, i64::MAX);
|
||||
assert_eq!(range.end, -1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_bounded_overflow() {
|
||||
// Overflowing bounded range clamps start/end to `i64::MAX`.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Bounded(u64::MAX - 10..u64::MAX));
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!(range.start, i64::MAX);
|
||||
assert_eq!(range.end, i64::MAX);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_range_output_can_convert_field_delimiter() {
|
||||
let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);
|
||||
|
||||
@@ -596,9 +596,205 @@ pub struct PendingScannerHeal {
|
||||
pub last_admission_reason: String,
|
||||
}
|
||||
|
||||
/// One bucket's standing in its set's service order. The wait position, the
|
||||
/// quanta granted and the last certification are separate quantities: a bucket
|
||||
/// that keeps failing moves to the tail like any other and only adds debt.
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketService {
|
||||
#[serde(default)]
|
||||
pub waiting_since: u64,
|
||||
#[serde(default)]
|
||||
pub service_quanta: u64,
|
||||
#[serde(default)]
|
||||
pub last_certified: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub failure_debt: u64,
|
||||
#[serde(default)]
|
||||
pub incarnation: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
// Nested derives encode positionally; a map lets later fields stay optional.
|
||||
impl Serialize for DataUsageBucketService {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let field_count = 2
|
||||
+ usize::from(self.last_certified.is_some())
|
||||
+ usize::from(self.failure_debt != 0)
|
||||
+ usize::from(self.incarnation.is_some());
|
||||
let mut map = serializer.serialize_map(Some(field_count))?;
|
||||
map.serialize_entry("waiting_since", &self.waiting_since)?;
|
||||
map.serialize_entry("service_quanta", &self.service_quanta)?;
|
||||
if let Some(last_certified) = self.last_certified {
|
||||
map.serialize_entry("last_certified", &last_certified)?;
|
||||
}
|
||||
if self.failure_debt != 0 {
|
||||
map.serialize_entry("failure_debt", &self.failure_debt)?;
|
||||
}
|
||||
if let Some(incarnation) = self.incarnation {
|
||||
map.serialize_entry("incarnation", &incarnation)?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable first-in-first-out service order of one set, in generations: every
|
||||
/// bucket at or below `generation_floor` still waits for its turn and everything
|
||||
/// above it was served, so a new bucket queues behind waiters but ahead of
|
||||
/// buckets that already had a turn. It only orders work:
|
||||
/// no entry proves coverage, authorizes a skip or releases a responsibility,
|
||||
/// so losing it restarts fairness and nothing else. Like the set cache's bucket
|
||||
/// entries and incarnations, it holds one record per bucket of the inventory.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketServiceLedger {
|
||||
/// Movement epoch the order was built under; another epoch invalidates it.
|
||||
#[serde(default)]
|
||||
pub publication_epoch: u64,
|
||||
/// Leader that saved the order; an incomplete save stamps nothing else.
|
||||
#[serde(default)]
|
||||
pub leader_epoch: u64,
|
||||
#[serde(default)]
|
||||
pub next_sequence: u64,
|
||||
#[serde(default)]
|
||||
pub generation_floor: u64,
|
||||
#[serde(default)]
|
||||
pub buckets: HashMap<String, DataUsageBucketService>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageBucketServiceLedger {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut map = serializer.serialize_map(Some(5))?;
|
||||
map.serialize_entry("publication_epoch", &self.publication_epoch)?;
|
||||
map.serialize_entry("leader_epoch", &self.leader_epoch)?;
|
||||
map.serialize_entry("next_sequence", &self.next_sequence)?;
|
||||
map.serialize_entry("generation_floor", &self.generation_floor)?;
|
||||
map.serialize_entry("buckets", &self.buckets)?;
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageBucketServiceLedger {
|
||||
/// Rebuilds the order for this cycle's inventory. Decoded state that fails
|
||||
/// validation or belongs to another movement epoch starts a new order;
|
||||
/// a bucket whose incarnation changed is a new arrival behind all waiters.
|
||||
pub(crate) fn restore<'a>(
|
||||
persisted: Option<Self>,
|
||||
publication_epoch: u64,
|
||||
inventory: impl Iterator<Item = &'a str>,
|
||||
incarnations: Option<&HashMap<String, uuid::Uuid>>,
|
||||
) -> Self {
|
||||
let mut ledger = persisted
|
||||
.filter(|ledger| ledger.publication_epoch == publication_epoch && ledger.is_valid())
|
||||
.unwrap_or_else(|| Self {
|
||||
publication_epoch,
|
||||
..Default::default()
|
||||
});
|
||||
let previous = std::mem::take(&mut ledger.buckets);
|
||||
let mut arrivals = Vec::new();
|
||||
for bucket in inventory {
|
||||
let incarnation = incarnations.and_then(|current| current.get(bucket)).copied();
|
||||
match previous.get(bucket) {
|
||||
Some(service) if service.incarnation.is_none() || incarnation.is_none() || service.incarnation == incarnation => {
|
||||
ledger.buckets.insert(
|
||||
bucket.to_owned(),
|
||||
DataUsageBucketService {
|
||||
incarnation: incarnation.or(service.incarnation),
|
||||
..*service
|
||||
},
|
||||
);
|
||||
}
|
||||
_ => arrivals.push((bucket, incarnation)),
|
||||
}
|
||||
}
|
||||
if arrivals.is_empty() {
|
||||
let floor = ledger.generation_floor;
|
||||
if ledger.buckets.values().all(|service| service.waiting_since > floor) {
|
||||
ledger.generation_floor = ledger.next_sequence;
|
||||
}
|
||||
return ledger;
|
||||
}
|
||||
// Open one position between the waiters and the served buckets.
|
||||
let floor = ledger.generation_floor;
|
||||
ledger.generation_floor = floor.saturating_add(1);
|
||||
ledger.next_sequence = ledger.next_sequence.saturating_add(1);
|
||||
for service in ledger.buckets.values_mut().filter(|service| service.waiting_since > floor) {
|
||||
service.waiting_since = service.waiting_since.saturating_add(1);
|
||||
}
|
||||
for (bucket, incarnation) in arrivals {
|
||||
ledger.buckets.insert(
|
||||
bucket.to_owned(),
|
||||
DataUsageBucketService {
|
||||
waiting_since: ledger.generation_floor,
|
||||
incarnation,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
ledger
|
||||
}
|
||||
|
||||
fn is_valid(&self) -> bool {
|
||||
self.generation_floor <= self.next_sequence
|
||||
&& self
|
||||
.buckets
|
||||
.iter()
|
||||
.all(|(bucket, service)| !bucket.is_empty() && service.waiting_since <= self.next_sequence)
|
||||
}
|
||||
|
||||
/// Smaller positions have waited longer; an untracked bucket queues last.
|
||||
pub(crate) fn wait_position(&self, bucket: &str) -> u64 {
|
||||
self.buckets.get(bucket).map_or(u64::MAX, |service| service.waiting_since)
|
||||
}
|
||||
|
||||
/// Moves a bucket to the tail and returns its previous wait position.
|
||||
pub(crate) fn record_service(&mut self, bucket: &str) -> Option<u64> {
|
||||
let service = self.buckets.get_mut(bucket)?;
|
||||
self.next_sequence = self.next_sequence.saturating_add(1);
|
||||
let previous = std::mem::replace(&mut service.waiting_since, self.next_sequence);
|
||||
service.service_quanta = service.service_quanta.saturating_add(1);
|
||||
Some(previous)
|
||||
}
|
||||
|
||||
/// Undoes `record_service` for an admission a resource veto turned back.
|
||||
pub(crate) fn return_service(&mut self, bucket: &str, waiting_since: u64) {
|
||||
if let Some(service) = self.buckets.get_mut(bucket) {
|
||||
service.waiting_since = waiting_since;
|
||||
service.service_quanta = service.service_quanta.saturating_sub(1);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_certified(&mut self, bucket: &str) {
|
||||
if let Some(service) = self.buckets.get_mut(bucket) {
|
||||
service.last_certified = Some(service.waiting_since);
|
||||
service.failure_debt = 0;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_failure(&mut self, bucket: &str) {
|
||||
if let Some(service) = self.buckets.get_mut(bucket) {
|
||||
service.failure_debt = service.failure_debt.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Buckets holding unpaid failures and the first of them by name.
|
||||
pub(crate) fn failure_debt(&self) -> (usize, Option<&str>) {
|
||||
let mut debtors = self
|
||||
.buckets
|
||||
.iter()
|
||||
.filter(|(_, service)| service.failure_debt > 0)
|
||||
.map(|(bucket, _)| bucket.as_str());
|
||||
let first = debtors.next();
|
||||
let (count, first) = debtors.fold((usize::from(first.is_some()), first), |(count, first), bucket| {
|
||||
(count + 1, first.min(Some(bucket)))
|
||||
});
|
||||
(count, first)
|
||||
}
|
||||
}
|
||||
|
||||
/// Data usage cache info
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageCacheInfo {
|
||||
/// In-memory only: this partial stopped on admission pressure, not a failure.
|
||||
#[serde(skip)]
|
||||
pub scan_resource_deferred: bool,
|
||||
pub name: String,
|
||||
pub next_cycle: u64,
|
||||
pub last_update: Option<SystemTime>,
|
||||
@@ -615,6 +811,11 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
/// Indexes of the other directories on the active walk path. A restart
|
||||
/// must replay every wide ancestor before it reaches the selected one, and
|
||||
/// recharging those replays would stop a fixed budget from ever advancing.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub scan_raw_enumeration_path_page_indexes: Vec<RawEnumerationPageIndex>,
|
||||
#[serde(default)]
|
||||
pub scan_identity: Option<DataUsageScanIdentity>,
|
||||
#[serde(default)]
|
||||
@@ -674,6 +875,9 @@ pub struct DataUsageCacheInfo {
|
||||
/// skipping an unselected bucket in a later scoped set scan.
|
||||
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
|
||||
pub scan_bucket_incarnations: HashMap<String, uuid::Uuid>,
|
||||
/// Bucket service order of a set cache; absent from bucket caches.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_bucket_service: Option<DataUsageBucketServiceLedger>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageCacheInfo {
|
||||
@@ -686,6 +890,7 @@ impl Serialize for DataUsageCacheInfo {
|
||||
let field_count = 16
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
|
||||
+ usize::from(!self.scan_raw_enumeration_path_page_indexes.is_empty())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_coverage_receipt.is_some())
|
||||
@@ -699,7 +904,8 @@ impl Serialize for DataUsageCacheInfo {
|
||||
+ usize::from(self.lkg_scan_plan_digest.is_some())
|
||||
+ usize::from(self.scan_execution_digest.is_some())
|
||||
+ usize::from(self.segment_invalidation_proof.is_some())
|
||||
+ usize::from(!self.scan_bucket_incarnations.is_empty());
|
||||
+ usize::from(!self.scan_bucket_incarnations.is_empty())
|
||||
+ usize::from(self.scan_bucket_service.is_some());
|
||||
let mut state = serializer.serialize_map(Some(field_count))?;
|
||||
state.serialize_entry("name", &self.name)?;
|
||||
state.serialize_entry("next_cycle", &self.next_cycle)?;
|
||||
@@ -717,6 +923,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(index) = &self.scan_raw_enumeration_page_index {
|
||||
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
|
||||
}
|
||||
if !self.scan_raw_enumeration_path_page_indexes.is_empty() {
|
||||
state.serialize_entry("scan_raw_enumeration_path_page_indexes", &self.scan_raw_enumeration_path_page_indexes)?;
|
||||
}
|
||||
if let Some(identity) = self.scan_identity {
|
||||
state.serialize_entry("scan_identity", &identity)?;
|
||||
}
|
||||
@@ -765,6 +974,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if !self.scan_bucket_incarnations.is_empty() {
|
||||
state.serialize_entry("scan_bucket_incarnations", &self.scan_bucket_incarnations)?;
|
||||
}
|
||||
if let Some(ledger) = &self.scan_bucket_service {
|
||||
state.serialize_entry("scan_bucket_service", ledger)?;
|
||||
}
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -902,6 +1114,30 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Identifies how far durable resume state has advanced. Discovery alone
|
||||
/// does not change it; only committed raw pages or coverage frontier do.
|
||||
pub(crate) fn durable_scan_position(&self) -> (Option<String>, Option<String>, u64) {
|
||||
let raw_entries = self
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.iter()
|
||||
.chain(&self.info.scan_raw_enumeration_path_page_indexes)
|
||||
.map(|index| match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => 0,
|
||||
RawEnumerationPageOwnerStatus::Building { indexed_entries, .. }
|
||||
| RawEnumerationPageOwnerStatus::Ready { indexed_entries, .. } => indexed_entries,
|
||||
})
|
||||
.fold(0, u64::saturating_add);
|
||||
(
|
||||
self.info.scan_resume_after.clone(),
|
||||
self.info
|
||||
.scan_coverage_receipt
|
||||
.as_ref()
|
||||
.map(|receipt| receipt.through.clone()),
|
||||
raw_entries,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn has_scan_position(&self) -> bool {
|
||||
self.info.scan_resume_after.is_some()
|
||||
|| self.info.scan_checkpoint.is_some()
|
||||
@@ -1044,7 +1280,14 @@ impl DataUsageCache {
|
||||
}
|
||||
if self.validated_raw_enumeration_page_index().is_none() {
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
}
|
||||
let path_indexes = std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes);
|
||||
self.info.scan_raw_enumeration_path_page_indexes = path_indexes
|
||||
.into_iter()
|
||||
.take(MAX_DATA_USAGE_CACHE_DEPTH)
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
.collect();
|
||||
let cursor_is_valid = !self.has_scan_position()
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
@@ -1087,6 +1330,7 @@ impl DataUsageCache {
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -1230,22 +1474,37 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub(crate) fn validated_raw_enumeration_page_index(&self) -> Option<&RawEnumerationPageIndex> {
|
||||
let index = self.info.scan_raw_enumeration_page_index.as_ref()?;
|
||||
self.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.as_ref()
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
}
|
||||
|
||||
/// The durable index of one directory on the saved walk path.
|
||||
pub(crate) fn validated_raw_enumeration_page_index_for_parent(&self, parent: &str) -> Option<&RawEnumerationPageIndex> {
|
||||
self.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.iter()
|
||||
.chain(&self.info.scan_raw_enumeration_path_page_indexes)
|
||||
.find(|index| index.parent() == Some(parent))
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
}
|
||||
|
||||
fn raw_enumeration_page_index_is_valid(&self, index: &RawEnumerationPageIndex) -> bool {
|
||||
if self.info.scan_progress.is_none()
|
||||
|| !self.info.scan_identity.is_some_and(|identity| identity.is_valid())
|
||||
|| self.info.source.is_none()
|
||||
|| index.committed_entries().is_err()
|
||||
|| index.indexed_entries().is_err()
|
||||
{
|
||||
return None;
|
||||
return false;
|
||||
}
|
||||
let parent = match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => return None,
|
||||
match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => false,
|
||||
RawEnumerationPageOwnerStatus::Building { parent, .. } | RawEnumerationPageOwnerStatus::Ready { parent, .. } => {
|
||||
parent
|
||||
path_is_in_bucket_scope(&self.info.name, &parent)
|
||||
}
|
||||
};
|
||||
path_is_in_bucket_scope(&self.info.name, &parent).then_some(index)
|
||||
}
|
||||
}
|
||||
|
||||
/// Seal only the frontier supplied by completed traversal, never a restored cursor.
|
||||
|
||||
@@ -57,6 +57,66 @@ fn scoped_scan_coverage_metadata_preserves_map_compatibility() {
|
||||
assert_eq!(decoded.scan_coverage_digest, Some(TEST_PLAN_DIGEST));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_path_indexes_preserve_map_compatibility_and_validation() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct LegacyInfo {
|
||||
name: String,
|
||||
scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
}
|
||||
let index_for = |parent: &str| RawEnumerationPageIndex::new(parent, 8).expect("fixture index");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
scan_progress: Some(DataUsageScanProgress {
|
||||
started_plan: TEST_PLAN_DIGEST,
|
||||
requested_plan: TEST_PLAN_DIGEST,
|
||||
}),
|
||||
scan_identity: Some(DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(1),
|
||||
set_layout: DataUsageScanPlanDigest([2; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
}),
|
||||
source: Some(DataUsageCacheSource::new(0, 0)),
|
||||
scan_raw_enumeration_page_index: Some(index_for("bucket/wide")),
|
||||
scan_raw_enumeration_path_page_indexes: vec![index_for("bucket"), index_for("other-bucket/wide")],
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
};
|
||||
assert!(cache.validated_raw_enumeration_page_index_for_parent("bucket").is_some());
|
||||
assert!(
|
||||
cache
|
||||
.validated_raw_enumeration_page_index_for_parent("other-bucket/wide")
|
||||
.is_none(),
|
||||
"a saved path index must stay inside its bucket"
|
||||
);
|
||||
|
||||
let encoded = rmp_serde::to_vec(&cache.info).expect("path indexes should remain map encoded");
|
||||
let legacy: LegacyInfo = rmp_serde::from_slice(&encoded).expect("old map readers should ignore the path indexes");
|
||||
assert_eq!(legacy.name, "bucket");
|
||||
assert_eq!(
|
||||
legacy
|
||||
.scan_raw_enumeration_page_index
|
||||
.and_then(|index| index.parent().map(str::to_owned)),
|
||||
Some("bucket/wide".to_string())
|
||||
);
|
||||
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&encoded).expect("new reader should restore the path indexes");
|
||||
assert_eq!(decoded.scan_raw_enumeration_path_page_indexes.len(), 2);
|
||||
|
||||
cache.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
let encoded = rmp_serde::to_vec(&cache.info).expect("empty path indexes should encode");
|
||||
assert!(
|
||||
!encoded
|
||||
.windows(b"scan_raw_enumeration_path_page_indexes".len())
|
||||
.any(|window| window == b"scan_raw_enumeration_path_page_indexes"),
|
||||
"the default must stay absent from the map"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct CachePutRecord {
|
||||
object: String,
|
||||
|
||||
@@ -80,6 +80,7 @@ impl RemoteScannerBudget {
|
||||
max_duration: self.max_duration_ms.map(Duration::from_millis),
|
||||
max_objects: self.max_objects,
|
||||
max_directories: self.max_directories,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,6 +743,7 @@ pub(crate) fn lookup_scanner_runtime_config(
|
||||
max_duration: cycle_max_duration.filter(|duration| !duration.is_zero()),
|
||||
max_objects: cycle_max_objects,
|
||||
max_directories: cycle_max_directories,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (bitrot_cycle, bitrot_cycle_source) = if let Some(value) = rustfs_utils::get_env_opt_str(ENV_SCANNER_BITROT_CYCLE_SECS) {
|
||||
@@ -1171,6 +1172,7 @@ mod tests {
|
||||
max_duration: Some(Duration::from_secs(1800)),
|
||||
max_objects: Some(1000),
|
||||
max_directories: Some(8000),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -474,14 +474,17 @@ fn scan_cycle_partial_reason(reason: Option<ScannerCycleBudgetReason>) -> ScanCy
|
||||
match reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => ScanCyclePartialReason::Runtime,
|
||||
Some(ScannerCycleBudgetReason::Objects) => ScanCyclePartialReason::Objects,
|
||||
Some(ScannerCycleBudgetReason::Directories) => ScanCyclePartialReason::Directories,
|
||||
// Raw enumeration is directory work; the reason set is a stable metric label.
|
||||
Some(ScannerCycleBudgetReason::Directories | ScannerCycleBudgetReason::Entries) => ScanCyclePartialReason::Directories,
|
||||
None => ScanCyclePartialReason::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_cycle_partial_source(reason: Option<ScannerCycleBudgetReason>) -> Option<ScannerWorkSource> {
|
||||
match reason {
|
||||
Some(ScannerCycleBudgetReason::Objects | ScannerCycleBudgetReason::Directories) => Some(ScannerWorkSource::Usage),
|
||||
Some(ScannerCycleBudgetReason::Objects | ScannerCycleBudgetReason::Directories | ScannerCycleBudgetReason::Entries) => {
|
||||
Some(ScannerWorkSource::Usage)
|
||||
}
|
||||
Some(ScannerCycleBudgetReason::Runtime) | None => None,
|
||||
}
|
||||
}
|
||||
@@ -1059,6 +1062,31 @@ fn prepare_cycle_for_usage_floor_bootstrap(
|
||||
}
|
||||
}
|
||||
|
||||
/// Overwrite an unreadable usage floor with a bootstrap marker using the
|
||||
/// file's current revision for CAS. A successful publish leaves a
|
||||
/// `BootstrapPending` floor that the normal load path picks up on retry.
|
||||
/// Returns `Err` when the floor is not even readable (transient I/O) so the
|
||||
/// caller keeps the supervisor-retry behavior instead of rebuilding.
|
||||
async fn rebuild_unreadable_usage_floor(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
) -> Result<(), ScannerError> {
|
||||
let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
|
||||
return Err(ScannerError::Other("scanner usage floor rebuild is blocked by data movement".to_string()));
|
||||
};
|
||||
let (_data, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("scanner usage floor revision unreadable for rebuild: {err}")))?;
|
||||
publish_scanner_usage_bootstrap_primary(
|
||||
storeapi,
|
||||
&revision,
|
||||
expected_epoch,
|
||||
None,
|
||||
ScannerUsageBootstrapPublishContext::Recovery,
|
||||
|| true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
init_data_scanner_with_storage(ctx, storeapi, ScannerRuntimeGuard::new()).await;
|
||||
}
|
||||
@@ -3087,6 +3115,27 @@ where
|
||||
Err(err) => usage_floor_result = Err(err),
|
||||
}
|
||||
}
|
||||
// backlog#2918: after dual-corruption recovery has been attempted an
|
||||
// unreadable floor is no longer a quota proof. Report the failure, then
|
||||
// rebuild through a CAS overwrite before giving up. Transient I/O errors
|
||||
// fail the rebuild and fall through to the supervisor-retry stop below.
|
||||
if usage_floor_result.is_err() {
|
||||
let error = usage_floor_result.as_ref().unwrap_err().to_string();
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "usage_floor_load_failed",
|
||||
error = %error,
|
||||
"Scanner usage floor is unreadable after recovery; attempting rebuild"
|
||||
);
|
||||
record_scanner_usage_floor_failure(error);
|
||||
if rebuild_unreadable_usage_floor(storeapi.clone()).await.is_ok() {
|
||||
usage_floor_result = persisted_usage_floor_for_startup(storeapi.clone(), true).await;
|
||||
}
|
||||
}
|
||||
let (usage_floor, usage_floor_startup) = match usage_floor_result {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
@@ -3099,7 +3148,7 @@ where
|
||||
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
state = "usage_floor_load_failed",
|
||||
error = %err,
|
||||
"Scanner stopped because the persisted usage floor could not be loaded"
|
||||
"Scanner usage floor could not be loaded or rebuilt; stopping leader iteration for supervisor retry"
|
||||
);
|
||||
record_scanner_usage_floor_failure(error.clone());
|
||||
global_metrics().set_cycle(None).await;
|
||||
|
||||
@@ -41,20 +41,6 @@ fn reset_recovery_status() {
|
||||
.expect("test must be able to reset the local recovery observation");
|
||||
}
|
||||
|
||||
async fn wait_for_recovery_attempt(attempt: u32) -> ScannerCycleRecoveryStatus {
|
||||
tokio::time::timeout(WAIT_BUDGET, async {
|
||||
loop {
|
||||
let status = scanner_cycle_recovery_status();
|
||||
if status.retry_count >= u64::from(attempt) {
|
||||
return status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("recovery attempt {attempt} was never recorded: {:?}", scanner_cycle_recovery_status()))
|
||||
}
|
||||
|
||||
async fn wait_for_recovery_state(state: &str) -> ScannerCycleRecoveryStatus {
|
||||
tokio::time::timeout(WAIT_BUDGET, async {
|
||||
loop {
|
||||
@@ -130,20 +116,19 @@ fn started_rebalance() -> EcstoreRebalanceMeta {
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#8670: a usage floor that is present but untrusted must not
|
||||
/// drive the supervisor into a terminal `paused, retryable=false` observation.
|
||||
/// Every attempt stays retryable with a bounded backoff, and repairing the
|
||||
/// floor recovers the scanner without a process restart.
|
||||
/// rustfs/rustfs#8670 + backlog#2918: a usage floor that is present but
|
||||
/// untrusted must not stop the scanner. The unreadable floor is reported and
|
||||
/// rebuilt through a CAS overwrite so the scanner recovers without a process
|
||||
/// restart or a terminal `paused, retryable=false` observation.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn usage_floor_load_failure_backs_off_and_recovers_without_restart() {
|
||||
async fn usage_floor_load_failure_rebuilds_and_recovers_without_restart() {
|
||||
temp_env::async_with_vars(SUPERVISOR_ENV, async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
reset_recovery_status();
|
||||
let (_dir, store) = setup_scanner_cycle_store().await;
|
||||
// Present but without a baseline identity: the shape #8670 reported,
|
||||
// whose error points the operator at the usage-state reset.
|
||||
// Present but without a baseline identity: the shape #8670 reported.
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
@@ -156,41 +141,11 @@ async fn usage_floor_load_failure_backs_off_and_recovers_without_restart() {
|
||||
let _cancel_on_drop = ctx.clone().drop_guard();
|
||||
init_data_scanner_with_storage(ctx.clone(), store.clone(), ScannerRuntimeGuard::new()).await;
|
||||
|
||||
let attempts = MAX_SCANNER_CYCLE_RECOVERY_RETRIES + 2;
|
||||
let mut first_detected = None;
|
||||
for attempt in 1..=attempts {
|
||||
let status = wait_for_recovery_attempt(attempt).await;
|
||||
assert_eq!(status.classification.as_deref(), Some("usage_floor_load_failed"), "attempt {attempt}");
|
||||
assert!(status.retryable, "attempt {attempt} must stay retryable: {status:?}");
|
||||
assert_ne!(status.state, "paused", "attempt {attempt} must not report a terminal pause");
|
||||
if attempt >= MAX_SCANNER_CYCLE_RECOVERY_RETRIES {
|
||||
assert_eq!(status.state, "backoff", "attempt {attempt}: {status:?}");
|
||||
} else {
|
||||
assert_eq!(status.state, "usage_floor_load_failed", "attempt {attempt}: {status:?}");
|
||||
}
|
||||
assert!(
|
||||
status
|
||||
.reason
|
||||
.as_deref()
|
||||
.is_some_and(|reason| reason.contains("usage-state/reset") && reason.contains("next automatic attempt")),
|
||||
"the operator hint and the next attempt must both be visible: {status:?}"
|
||||
);
|
||||
assert!(status.next_attempt_at_unix_secs.is_some(), "attempt {attempt}: {status:?}");
|
||||
match first_detected {
|
||||
None => first_detected = status.first_detected_at_unix_secs,
|
||||
Some(first) => assert_eq!(status.first_detected_at_unix_secs, Some(first), "attempt {attempt}"),
|
||||
}
|
||||
if attempt == attempts {
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), valid_usage_floor(0, 0))
|
||||
.await
|
||||
.expect("repaired usage floor should persist");
|
||||
}
|
||||
notify_scanner_cycle_recovery_wake();
|
||||
}
|
||||
|
||||
// backlog#2918: the unreadable floor is rebuilt automatically; the
|
||||
// scanner must reach healthy without a manual repair or terminal pause.
|
||||
let healthy = wait_for_recovery_state("healthy").await;
|
||||
assert_eq!(healthy.retry_count, 0, "a successful attempt clears the retry budget: {healthy:?}");
|
||||
assert!(!healthy.retryable);
|
||||
assert!(!healthy.retryable, "rebuild must clear the retry budget: {healthy:?}");
|
||||
assert_ne!(healthy.state, "paused", "rebuild must not report a terminal pause");
|
||||
wait_for_persisted_cycle(&store, |cycle, _| cycle.next >= 1).await;
|
||||
assert_eq!(scanner_cycle_schedule_status().execution_role, "leader");
|
||||
stop_supervisor(ctx).await;
|
||||
|
||||
@@ -24,13 +24,51 @@ const BUDGET_REASON_NONE: u8 = 0;
|
||||
const BUDGET_REASON_RUNTIME: u8 = 1;
|
||||
const BUDGET_REASON_OBJECTS: u8 = 2;
|
||||
const BUDGET_REASON_DIRECTORIES: u8 = 3;
|
||||
const BUDGET_REASON_ENTRIES: u8 = 4;
|
||||
const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128;
|
||||
/// One raw-enumeration page, so a directory stop always commits durable work.
|
||||
const DIRECTORY_ENTRY_BUDGET_FLOOR: u64 = 128;
|
||||
/// Directory entries one object or directory budget unit pays for. A forward
|
||||
/// sweep classifies in sorted order and so needs a whole listing first; a
|
||||
/// directory a few times wider than the budget must still reach objects in
|
||||
/// its first round, while a far wider one builds its index across rounds.
|
||||
const DIRECTORY_ENTRIES_PER_COUNT_UNIT: u64 = 16;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerCycleBudgetConfig {
|
||||
pub max_duration: Option<Duration>,
|
||||
pub max_objects: Option<u64>,
|
||||
pub max_directories: Option<u64>,
|
||||
/// Raw directory entries first observed under this budget. Entries replayed
|
||||
/// from a validated page index are not charged.
|
||||
pub max_entries: Option<u64>,
|
||||
/// Cumulative name bytes of the charged raw entries.
|
||||
pub max_entry_name_bytes: Option<u64>,
|
||||
/// One bucket's share. `None` lets a bucket consume the whole budget.
|
||||
pub bucket_quantum: Option<ScannerBucketQuantum>,
|
||||
}
|
||||
|
||||
/// Limits of one bucket admission, charged to the owning cycle budget as well.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerBucketQuantum {
|
||||
pub max_duration: Option<Duration>,
|
||||
pub max_objects: Option<u64>,
|
||||
pub max_directories: Option<u64>,
|
||||
pub max_entries: Option<u64>,
|
||||
pub max_entry_name_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
impl From<ScannerBucketQuantum> for ScannerCycleBudgetConfig {
|
||||
fn from(quantum: ScannerBucketQuantum) -> Self {
|
||||
Self {
|
||||
max_duration: quantum.max_duration,
|
||||
max_objects: quantum.max_objects,
|
||||
max_directories: quantum.max_directories,
|
||||
max_entries: quantum.max_entries,
|
||||
max_entry_name_bytes: quantum.max_entry_name_bytes,
|
||||
bucket_quantum: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -38,6 +76,7 @@ pub(crate) enum ScannerCycleBudgetReason {
|
||||
Runtime,
|
||||
Objects,
|
||||
Directories,
|
||||
Entries,
|
||||
}
|
||||
|
||||
impl ScannerCycleBudgetReason {
|
||||
@@ -46,6 +85,7 @@ impl ScannerCycleBudgetReason {
|
||||
Self::Runtime => BUDGET_REASON_RUNTIME,
|
||||
Self::Objects => BUDGET_REASON_OBJECTS,
|
||||
Self::Directories => BUDGET_REASON_DIRECTORIES,
|
||||
Self::Entries => BUDGET_REASON_ENTRIES,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,12 +94,15 @@ impl ScannerCycleBudgetReason {
|
||||
BUDGET_REASON_RUNTIME => Some(Self::Runtime),
|
||||
BUDGET_REASON_OBJECTS => Some(Self::Objects),
|
||||
BUDGET_REASON_DIRECTORIES => Some(Self::Directories),
|
||||
BUDGET_REASON_ENTRIES => Some(Self::Entries),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ScannerCycleBudget {
|
||||
/// The cycle budget a bucket quantum also charges.
|
||||
parent: Option<Arc<ScannerCycleBudget>>,
|
||||
token: CancellationToken,
|
||||
reason: Arc<AtomicU8>,
|
||||
started_at: Instant,
|
||||
@@ -67,11 +110,17 @@ pub struct ScannerCycleBudget {
|
||||
max_duration: Option<Duration>,
|
||||
max_objects: Option<u64>,
|
||||
max_directories: Option<u64>,
|
||||
max_entries: Option<u64>,
|
||||
max_entry_name_bytes: Option<u64>,
|
||||
directory_entry_limit: Option<u64>,
|
||||
bucket_quantum: Option<ScannerBucketQuantum>,
|
||||
track_progress: bool,
|
||||
track_unbounded_counts: bool,
|
||||
objects_scanned: AtomicU64,
|
||||
directories_started: AtomicU64,
|
||||
entries_visited: AtomicU64,
|
||||
entries_charged: AtomicU64,
|
||||
entry_name_bytes_charged: AtomicU64,
|
||||
last_progress_millis: AtomicU64,
|
||||
cycle_state_persisted: AtomicBool,
|
||||
remote_scan_progress: Arc<RemoteScanProgress>,
|
||||
@@ -80,16 +129,16 @@ pub struct ScannerCycleBudget {
|
||||
impl ScannerCycleBudget {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
Self::new_inner(parent, config, false, false)
|
||||
Self::new_inner(parent, config, false, false, None)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
Self::new_inner(parent, config, true, true)
|
||||
Self::new_inner(parent, config, true, true, None)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
let track_progress = config.max_duration.is_some();
|
||||
Self::new_inner(parent, config, track_progress, false)
|
||||
Self::new_inner(parent, config, track_progress, false, None)
|
||||
}
|
||||
|
||||
fn new_inner(
|
||||
@@ -97,6 +146,7 @@ impl ScannerCycleBudget {
|
||||
config: ScannerCycleBudgetConfig,
|
||||
track_progress: bool,
|
||||
track_unbounded_counts: bool,
|
||||
cycle_budget: Option<Arc<ScannerCycleBudget>>,
|
||||
) -> Arc<Self> {
|
||||
let token = parent.child_token();
|
||||
let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
|
||||
@@ -124,7 +174,11 @@ impl ScannerCycleBudget {
|
||||
});
|
||||
}
|
||||
|
||||
let remote_scan_progress = cycle_budget
|
||||
.as_ref()
|
||||
.map_or_else(|| Arc::new(RemoteScanProgress::default()), |budget| budget.remote_scan_progress());
|
||||
Arc::new(Self {
|
||||
parent: cycle_budget,
|
||||
token,
|
||||
reason,
|
||||
started_at,
|
||||
@@ -132,14 +186,20 @@ impl ScannerCycleBudget {
|
||||
max_duration: config.max_duration,
|
||||
max_objects: config.max_objects,
|
||||
max_directories: config.max_directories,
|
||||
max_entries: config.max_entries,
|
||||
max_entry_name_bytes: config.max_entry_name_bytes,
|
||||
directory_entry_limit: directory_entry_limit(&config),
|
||||
bucket_quantum: config.bucket_quantum,
|
||||
track_progress,
|
||||
track_unbounded_counts,
|
||||
objects_scanned: AtomicU64::new(0),
|
||||
directories_started: AtomicU64::new(0),
|
||||
entries_visited: AtomicU64::new(0),
|
||||
entries_charged: AtomicU64::new(0),
|
||||
entry_name_bytes_charged: AtomicU64::new(0),
|
||||
last_progress_millis: AtomicU64::new(0),
|
||||
cycle_state_persisted: AtomicBool::new(false),
|
||||
remote_scan_progress: Arc::new(RemoteScanProgress::default()),
|
||||
remote_scan_progress,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -147,12 +207,31 @@ impl ScannerCycleBudget {
|
||||
self.token.clone()
|
||||
}
|
||||
|
||||
/// Budget and cancellation scope of one bucket scan. Without a quantum the
|
||||
/// bucket shares this budget; a quantum stops only its own scan token.
|
||||
pub(crate) fn for_bucket(self: &Arc<Self>, ctx: &CancellationToken) -> (Arc<Self>, CancellationToken) {
|
||||
let Some(quantum) = self.bucket_quantum else {
|
||||
return (self.clone(), ctx.child_token());
|
||||
};
|
||||
let budget = Self::new_inner(ctx, quantum.into(), self.track_progress, self.track_unbounded_counts, Some(self.clone()));
|
||||
let token = budget.token();
|
||||
(budget, token)
|
||||
}
|
||||
|
||||
/// True when only this bucket's share ended and the cycle can continue.
|
||||
pub(crate) fn quantum_exhausted(&self) -> bool {
|
||||
self.parent.as_ref().is_some_and(|parent| !parent.budget_elapsed())
|
||||
&& self.reason.load(Ordering::Relaxed) != BUDGET_REASON_NONE
|
||||
}
|
||||
|
||||
pub(crate) fn budget_elapsed(&self) -> bool {
|
||||
self.reason.load(Ordering::Relaxed) != BUDGET_REASON_NONE
|
||||
|| self.parent.as_ref().is_some_and(|parent| parent.budget_elapsed())
|
||||
}
|
||||
|
||||
pub(crate) fn reason(&self) -> Option<ScannerCycleBudgetReason> {
|
||||
ScannerCycleBudgetReason::from_code(self.reason.load(Ordering::Relaxed))
|
||||
.or_else(|| self.parent.as_ref().and_then(|parent| parent.reason()))
|
||||
}
|
||||
|
||||
pub(crate) fn max_duration(&self) -> Option<Duration> {
|
||||
@@ -176,7 +255,10 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn requires_serial_progress_accounting(&self) -> bool {
|
||||
self.max_objects.is_some() || self.max_directories.is_some()
|
||||
self.max_objects.is_some()
|
||||
|| self.max_directories.is_some()
|
||||
|| self.max_entries.is_some()
|
||||
|| self.max_entry_name_bytes.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn remaining_config(&self) -> ScannerCycleBudgetConfig {
|
||||
@@ -195,6 +277,13 @@ impl ScannerCycleBudget {
|
||||
max_directories: self
|
||||
.max_directories
|
||||
.map(|max| max.saturating_sub(self.directories_started.load(Ordering::Relaxed))),
|
||||
max_entries: self
|
||||
.max_entries
|
||||
.map(|max| max.saturating_sub(self.entries_charged.load(Ordering::Relaxed))),
|
||||
max_entry_name_bytes: self
|
||||
.max_entry_name_bytes
|
||||
.map(|max| max.saturating_sub(self.entry_name_bytes_charged.load(Ordering::Relaxed))),
|
||||
bucket_quantum: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,6 +323,9 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
|
||||
if let Some(parent) = &self.parent {
|
||||
parent.record_entries_visited(entries_visited);
|
||||
}
|
||||
if self.track_progress {
|
||||
let entries = saturating_fetch_add(&self.entries_visited, entries_visited);
|
||||
self.record_progress_sample(entries);
|
||||
@@ -245,7 +337,11 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
|
||||
self.remote_scan_progress.record(objects_scanned, directories_started);
|
||||
// A quantum shares the cycle's progress record; its parent writes it.
|
||||
match &self.parent {
|
||||
Some(parent) => parent.record_remote_progress(objects_scanned, directories_started),
|
||||
None => self.remote_scan_progress.record(objects_scanned, directories_started),
|
||||
}
|
||||
if self.track_progress || self.max_objects.is_some() {
|
||||
let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
|
||||
if self.track_progress {
|
||||
@@ -271,6 +367,9 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_after_unreported_remote_progress(&self) {
|
||||
if let Some(parent) = &self.parent {
|
||||
parent.cancel_after_unreported_remote_progress();
|
||||
}
|
||||
if self.max_objects.is_some() {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Objects);
|
||||
} else if self.max_directories.is_some() {
|
||||
@@ -279,6 +378,11 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn try_start_directory(&self) -> bool {
|
||||
let parent_allows = self.parent.as_ref().is_none_or(|parent| parent.try_start_directory());
|
||||
self.try_start_own_directory() && parent_allows
|
||||
}
|
||||
|
||||
fn try_start_own_directory(&self) -> bool {
|
||||
if self.max_directories.is_none() && !self.track_unbounded_counts {
|
||||
return true;
|
||||
}
|
||||
@@ -298,7 +402,35 @@ impl ScannerCycleBudget {
|
||||
true
|
||||
}
|
||||
|
||||
/// Charges one raw entry first observed in this scan. `directory_entries`
|
||||
/// counts the charged entries of the current directory: a count budget is
|
||||
/// a total work limit, so one listing cannot read past it.
|
||||
///
|
||||
/// Returns `false` after charging the entry that exhausts a limit; the
|
||||
/// caller stops with the entry already recorded in its durable index.
|
||||
pub(crate) fn try_charge_raw_entry(&self, name_bytes: u64, directory_entries: u64) -> bool {
|
||||
let parent_allows = self
|
||||
.parent
|
||||
.as_ref()
|
||||
.is_none_or(|parent| parent.try_charge_raw_entry(name_bytes, directory_entries));
|
||||
let entries = self
|
||||
.max_entries
|
||||
.is_some_and(|max| saturating_fetch_add(&self.entries_charged, 1) >= max);
|
||||
let name_bytes = self
|
||||
.max_entry_name_bytes
|
||||
.is_some_and(|max| saturating_fetch_add(&self.entry_name_bytes_charged, name_bytes) >= max);
|
||||
let directory = self.directory_entry_limit.is_some_and(|limit| directory_entries >= limit);
|
||||
if entries || name_bytes || directory {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Entries);
|
||||
return false;
|
||||
}
|
||||
parent_allows
|
||||
}
|
||||
|
||||
pub(crate) fn record_object_scanned(&self) {
|
||||
if let Some(parent) = &self.parent {
|
||||
parent.record_object_scanned();
|
||||
}
|
||||
if self.max_objects.is_none() && !self.track_unbounded_counts {
|
||||
return;
|
||||
}
|
||||
@@ -337,6 +469,21 @@ fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_entry_limit(config: &ScannerCycleBudgetConfig) -> Option<u64> {
|
||||
if config.max_objects.is_none() && config.max_directories.is_none() {
|
||||
return None;
|
||||
}
|
||||
let total = config
|
||||
.max_objects
|
||||
.unwrap_or(0)
|
||||
.saturating_add(config.max_directories.unwrap_or(0));
|
||||
Some(
|
||||
total
|
||||
.saturating_mul(DIRECTORY_ENTRIES_PER_COUNT_UNIT)
|
||||
.max(DIRECTORY_ENTRY_BUDGET_FLOOR),
|
||||
)
|
||||
}
|
||||
|
||||
fn directory_budget_exhausted(directories: u64, max_directories: u64) -> bool {
|
||||
// Saturation hides a remote max+1 update when the configured limit is the
|
||||
// largest representable counter. Treat that boundary as exhausted rather
|
||||
@@ -354,6 +501,127 @@ impl Drop for ScannerCycleBudget {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_budget_limits_entries_and_name_bytes_with_one_reason() {
|
||||
let parent = CancellationToken::new();
|
||||
let by_entries = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_entries: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(by_entries.try_charge_raw_entry(1, 1));
|
||||
assert!(!by_entries.try_charge_raw_entry(1, 2));
|
||||
assert_eq!(by_entries.reason(), Some(ScannerCycleBudgetReason::Entries));
|
||||
assert!(by_entries.token().is_cancelled());
|
||||
|
||||
let by_bytes = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_entry_name_bytes: Some(10),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(by_bytes.try_charge_raw_entry(9, 1));
|
||||
assert!(!by_bytes.try_charge_raw_entry(1, 2));
|
||||
assert_eq!(by_bytes.reason(), Some(ScannerCycleBudgetReason::Entries));
|
||||
assert!(!parent.is_cancelled(), "a budget stop must not cancel its parent token");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_budget_derives_a_directory_limit_only_from_count_budgets() {
|
||||
let parent = CancellationToken::new();
|
||||
let runtime_only = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default());
|
||||
assert!(runtime_only.try_charge_raw_entry(1, u64::MAX));
|
||||
|
||||
let small = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(small.try_charge_raw_entry(1, DIRECTORY_ENTRY_BUDGET_FLOOR - 1));
|
||||
assert!(!small.try_charge_raw_entry(1, DIRECTORY_ENTRY_BUDGET_FLOOR));
|
||||
|
||||
let large = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1000),
|
||||
max_directories: Some(24),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(large.try_charge_raw_entry(1, 1024 * DIRECTORY_ENTRIES_PER_COUNT_UNIT - 1));
|
||||
assert!(!large.try_charge_raw_entry(1, 1024 * DIRECTORY_ENTRIES_PER_COUNT_UNIT));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_bucket_share_stops_only_its_own_scan() {
|
||||
let ctx = CancellationToken::new();
|
||||
let cycle = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(3),
|
||||
bucket_quantum: Some(ScannerBucketQuantum {
|
||||
max_objects: Some(2),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (first, first_token) = cycle.for_bucket(&cycle.token());
|
||||
first.record_object_scanned();
|
||||
first.record_object_scanned();
|
||||
assert!(first.quantum_exhausted());
|
||||
assert!(first_token.is_cancelled());
|
||||
assert_eq!(first.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
assert!(!cycle.budget_elapsed(), "a bucket share leaves the cycle running");
|
||||
assert!(!cycle.token().is_cancelled());
|
||||
assert_eq!(cycle.progress().0, 2, "bucket work is charged to the cycle");
|
||||
|
||||
let (second, second_token) = cycle.for_bucket(&cycle.token());
|
||||
assert!(!second.budget_elapsed());
|
||||
second.record_object_scanned();
|
||||
assert!(cycle.budget_elapsed(), "the total budget still bounds every share");
|
||||
assert!(second_token.is_cancelled());
|
||||
assert!(second.budget_elapsed());
|
||||
assert!(!second.quantum_exhausted(), "a cycle stop is not a quantum handoff");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fair_prefix_quantum_bucket_share_runtime_expires_independently() {
|
||||
let ctx = CancellationToken::new();
|
||||
let cycle = ScannerCycleBudget::new(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
bucket_quantum: Some(ScannerBucketQuantum {
|
||||
max_duration: Some(Duration::ZERO),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (bucket, token) = cycle.for_bucket(&cycle.token());
|
||||
tokio::time::timeout(Duration::from_secs(5), token.cancelled())
|
||||
.await
|
||||
.expect("quantum deadline should cancel the bucket scan");
|
||||
assert_eq!(bucket.reason(), Some(ScannerCycleBudgetReason::Runtime));
|
||||
assert!(!cycle.budget_elapsed());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_without_a_quantum_shares_the_cycle_budget() {
|
||||
let ctx = CancellationToken::new();
|
||||
let cycle = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let (bucket, token) = cycle.for_bucket(&cycle.token());
|
||||
assert!(Arc::ptr_eq(&bucket, &cycle));
|
||||
cycle.cancel_for_runtime();
|
||||
assert!(token.is_cancelled());
|
||||
assert!(!bucket.quantum_exhausted());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_budget_cancels_child_token() {
|
||||
let parent = CancellationToken::new();
|
||||
@@ -441,6 +709,7 @@ mod tests {
|
||||
max_duration: Some(Duration::from_secs(60)),
|
||||
max_objects: Some(10),
|
||||
max_directories: Some(5),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -498,7 +498,10 @@ fn checkpoint_reason_from_budget(reason: Option<ScannerCycleBudgetReason>) -> Da
|
||||
match reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => DataUsageScanCheckpointReason::Runtime,
|
||||
Some(ScannerCycleBudgetReason::Objects) => DataUsageScanCheckpointReason::Objects,
|
||||
Some(ScannerCycleBudgetReason::Directories) => DataUsageScanCheckpointReason::Directories,
|
||||
// This enum is persisted and older readers reject unknown variants.
|
||||
Some(ScannerCycleBudgetReason::Directories | ScannerCycleBudgetReason::Entries) => {
|
||||
DataUsageScanCheckpointReason::Directories
|
||||
}
|
||||
None => DataUsageScanCheckpointReason::Unknown,
|
||||
}
|
||||
}
|
||||
@@ -1302,9 +1305,11 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) {
|
||||
/// Records one raw entry. Returns whether a durable index holds it, which
|
||||
/// is what makes stopping at this entry recoverable.
|
||||
fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) -> bool {
|
||||
if self.old_cache.info.scan_progress.is_none() {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
if let Some(position) = self
|
||||
.raw_enumeration_progress
|
||||
@@ -1317,18 +1322,14 @@ impl FolderScanner {
|
||||
self.raw_enumeration_progress
|
||||
.push(RawEnumerationProgress::new(parent, page_index));
|
||||
}
|
||||
if let Some(progress) = self.raw_enumeration_progress.last_mut() {
|
||||
self.raw_enumeration_progress.last_mut().is_some_and(|progress| {
|
||||
progress.record_entry(entry);
|
||||
}
|
||||
progress.page_index.is_some()
|
||||
})
|
||||
}
|
||||
|
||||
fn raw_enumeration_page_index_for_parent(&self, parent: &str) -> Option<&RawEnumerationPageIndex> {
|
||||
self.old_cache
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
.as_ref()
|
||||
.filter(|index| index.parent() == Some(parent))?;
|
||||
self.old_cache.validated_raw_enumeration_page_index()
|
||||
self.old_cache.validated_raw_enumeration_page_index_for_parent(parent)
|
||||
}
|
||||
|
||||
fn raw_enumeration_committed_entry_oracle(&self, parent: &str) -> HashSet<String> {
|
||||
@@ -1372,23 +1373,36 @@ impl FolderScanner {
|
||||
if self.raw_enumeration_progress.is_empty() {
|
||||
return (None, None);
|
||||
}
|
||||
let progress_index = self
|
||||
.raw_enumeration_progress
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
let progress_index = self.selected_raw_enumeration_progress().unwrap_or(0);
|
||||
let progress = self.raw_enumeration_progress.swap_remove(progress_index);
|
||||
self.raw_enumeration_progress.clear();
|
||||
(progress.cursor(), progress.page_index())
|
||||
}
|
||||
|
||||
fn peek_raw_enumeration_resume_state(&self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
fn selected_raw_enumeration_progress(&self) -> Option<usize> {
|
||||
self.raw_enumeration_progress
|
||||
.iter()
|
||||
.max_by_key(|progress| progress.checkpointable_entry_count())
|
||||
.map_or((None, None), |progress| (progress.cursor(), progress.page_index()))
|
||||
.enumerate()
|
||||
.max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
|
||||
.map(|(index, _)| index)
|
||||
}
|
||||
|
||||
/// The walk path except the directory the resume state selects.
|
||||
fn raw_enumeration_path_page_indexes(&self) -> Vec<RawEnumerationPageIndex> {
|
||||
let selected = self.selected_raw_enumeration_progress();
|
||||
self.raw_enumeration_progress
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(index, _)| Some(*index) != selected)
|
||||
.filter_map(|(_, progress)| progress.page_index())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn peek_raw_enumeration_resume_state(&self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
self.selected_raw_enumeration_progress().map_or((None, None), |index| {
|
||||
let progress = &self.raw_enumeration_progress[index];
|
||||
(progress.cursor(), progress.page_index())
|
||||
})
|
||||
}
|
||||
|
||||
async fn scan_child_folder(
|
||||
@@ -1480,6 +1494,7 @@ impl FolderScanner {
|
||||
snapshot.info.last_update = Some(SystemTime::now());
|
||||
snapshot.info.snapshot_complete = false;
|
||||
let (cursor, page_index) = self.peek_raw_enumeration_resume_state();
|
||||
snapshot.info.scan_raw_enumeration_path_page_indexes = self.raw_enumeration_path_page_indexes();
|
||||
snapshot.info.scan_raw_enumeration_cursor = cursor;
|
||||
snapshot.info.scan_raw_enumeration_page_index = page_index;
|
||||
snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).ok()?;
|
||||
@@ -1976,6 +1991,9 @@ impl FolderScanner {
|
||||
let mut last_entry_progress = Instant::now();
|
||||
let mut raw_enumeration_complete = false;
|
||||
let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name);
|
||||
let raw_replay_floor = u64::try_from(raw_enumeration_committed_entries.len()).unwrap_or(u64::MAX);
|
||||
let mut raw_entries_observed = 0_u64;
|
||||
let mut raw_entries_charged = 0_u64;
|
||||
|
||||
let mut candidate_unclassified = mode != FolderScanMode::Namespace;
|
||||
let mut deferred_metadata = None;
|
||||
@@ -2061,8 +2079,13 @@ impl FolderScanner {
|
||||
continue;
|
||||
}
|
||||
if !replayed_metadata {
|
||||
let raw_entry_consumed_by_owner_index = raw_enumeration_committed_entries.contains(&file_name);
|
||||
if !raw_entry_consumed_by_owner_index {
|
||||
// A restored index appends one pending name per entry past
|
||||
// its old size, so charging only that tail keeps each
|
||||
// budget unit durable even if the listing order changed.
|
||||
raw_entries_observed = raw_entries_observed.saturating_add(1);
|
||||
let raw_entry_is_new =
|
||||
raw_entries_observed > raw_replay_floor && !raw_enumeration_committed_entries.contains(&file_name);
|
||||
if raw_entry_is_new {
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
@@ -2074,7 +2097,18 @@ impl FolderScanner {
|
||||
last_entry_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
let raw_entry_is_durable = self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
// Object metadata and the entries behind it are charged to
|
||||
// the object budget; this one covers listing work.
|
||||
let object_entry = file_name == STORAGE_FORMAT_FILE || deferred_metadata.is_some();
|
||||
if raw_entry_is_new && raw_entry_is_durable && !object_entry {
|
||||
raw_entries_charged = raw_entries_charged.saturating_add(1);
|
||||
let name_bytes = u64::try_from(file_name.len()).unwrap_or(u64::MAX);
|
||||
if !self.budget.try_charge_raw_entry(name_bytes, raw_entries_charged) {
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
}
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
}
|
||||
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
|
||||
@@ -2474,7 +2508,9 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
if raw_enumeration_complete {
|
||||
// A directory with queued children keeps its index until they end:
|
||||
// a restart replays this listing to reach the unfinished child.
|
||||
if raw_enumeration_complete && queued_folders.is_empty() {
|
||||
self.finish_raw_enumeration_parent(&folder.name);
|
||||
}
|
||||
|
||||
@@ -2683,10 +2719,12 @@ impl FolderScanner {
|
||||
"Scanner child folder scan failed"
|
||||
);
|
||||
self.coverage_gap |= forward_sweep;
|
||||
self.finish_raw_enumeration_parent(&folder_item.name);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
self.finish_raw_enumeration_parent(&folder_item.name);
|
||||
if outcome == FolderScanOutcome::Vanished {
|
||||
self.new_cache.delete_recursive(&h);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
@@ -3405,6 +3443,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
new_cache.info.scan_checkpoint = None;
|
||||
new_cache.info.scan_raw_enumeration_cursor = None;
|
||||
new_cache.info.scan_raw_enumeration_page_index = None;
|
||||
new_cache.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
if had_scan_checkpoint {
|
||||
global_metrics().record_scanner_checkpoint_cleared();
|
||||
@@ -3443,6 +3482,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
let root_hash = hash_path(&bucket_name);
|
||||
let root_has_progress = data_usage_root_has_progress(&root);
|
||||
let pending_heals_changed = scanner.pending_heals_changed;
|
||||
let raw_enumeration_path_page_indexes = scanner.raw_enumeration_path_page_indexes();
|
||||
let (raw_enumeration_cursor, raw_enumeration_page_index) = scanner.take_raw_enumeration_resume_state();
|
||||
let carry_forward_cache = ((raw_enumeration_cursor.is_some() || raw_enumeration_page_index.is_some())
|
||||
&& !root_has_progress)
|
||||
@@ -3465,6 +3505,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
}
|
||||
if raw_enumeration_page_index.is_some() {
|
||||
new_cache.info.scan_raw_enumeration_page_index = raw_enumeration_page_index;
|
||||
new_cache.info.scan_raw_enumeration_path_page_indexes = raw_enumeration_path_page_indexes;
|
||||
new_cache.info.scan_checkpoint = None;
|
||||
new_cache.info.scan_resume_after = None;
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
@@ -3495,6 +3536,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
global_metrics().record_scanner_checkpoint_cleared();
|
||||
}
|
||||
}
|
||||
new_cache.info.scan_resource_deferred = matches!(&e, ScannerError::ResourceDeferred);
|
||||
close_disk_guard.close().await;
|
||||
return Err(ScannerError::PartialCache(Box::new(std::mem::take(new_cache))));
|
||||
}
|
||||
@@ -3524,6 +3566,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
.insert(hash_path(&bucket_name).0, DataUsageEntry::default());
|
||||
}
|
||||
partial_cache.info.snapshot_complete = false;
|
||||
partial_cache.info.scan_resource_deferred = true;
|
||||
close_disk_guard.close().await;
|
||||
return Err(ScannerError::PartialCache(Box::new(partial_cache)));
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ use std::sync::Mutex;
|
||||
|
||||
pub(super) mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
mod fair_prefix_quantum;
|
||||
mod incremental_enumeration;
|
||||
mod listing_completion;
|
||||
mod metadata_visits;
|
||||
|
||||
@@ -51,6 +51,10 @@ impl Drop for ObservationGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn install_raw_entry_observer(root: PathBuf) -> ObservationGuard {
|
||||
install_raw_entry_budget(root, u64::MAX)
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn install_raw_entry_budget(root: PathBuf, limit: u64) -> ObservationGuard {
|
||||
*OBSERVATION.lock().expect("install raw-entry observation") = Some(Observation {
|
||||
root,
|
||||
@@ -83,6 +87,45 @@ pub(super) async fn read_bounded(path: &Path) -> Vec<u8> {
|
||||
bytes
|
||||
}
|
||||
|
||||
/// Creates a flat synthetic bucket and the checkpointable cache a first sweep
|
||||
/// starts from.
|
||||
pub(super) async fn seed_flat_bucket(disk_root: &Path, objects: usize) -> DataUsageCache {
|
||||
tokio::fs::create_dir(disk_root).await.expect("create fresh synthetic disk");
|
||||
for index in 0..objects {
|
||||
let object = format!("object-{index:04}");
|
||||
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
|
||||
let bytes = metadata_for_object_version("bucket", &object, Some(version));
|
||||
write_test_object_metadata_bytes(disk_root, "bucket", &object, &bytes).await;
|
||||
}
|
||||
let mut initial = DataUsageCache::default();
|
||||
initial.info.name = "bucket".to_string();
|
||||
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
|
||||
let plan = crate::data_usage_define::DataUsageScanPlanDigest([31; 32]);
|
||||
let identity = crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::from_u128(31),
|
||||
set_layout: crate::data_usage_define::DataUsageScanPlanDigest([32; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
assert_eq!(
|
||||
initial.prepare_bucket_checkpoint("bucket", 1, 0, source, plan, identity),
|
||||
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
initial.info.skip_healing = true;
|
||||
initial.replace("bucket", "", DataUsageEntry::default());
|
||||
initial
|
||||
}
|
||||
|
||||
pub(super) fn observed_raw_entries() -> u64 {
|
||||
OBSERVATION
|
||||
.lock()
|
||||
.expect("read raw-entry observation")
|
||||
.as_ref()
|
||||
.map_or(0, |observation| observation.entries)
|
||||
}
|
||||
|
||||
async fn round(request: &Request) -> serde_json::Value {
|
||||
assert!((1..=1024).contains(&request.objects));
|
||||
assert!((1..=4096).contains(&request.raw_entry_budget));
|
||||
@@ -90,31 +133,7 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
let disk_root = request.workspace.join("disk");
|
||||
let cache_path = request.workspace.join("cache.bin");
|
||||
if request.round == 0 {
|
||||
tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk");
|
||||
for index in 0..request.objects {
|
||||
let object = format!("object-{index:04}");
|
||||
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
|
||||
let bytes = metadata_for_object_version("bucket", &object, Some(version));
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await;
|
||||
}
|
||||
let mut initial = DataUsageCache::default();
|
||||
initial.info.name = "bucket".to_string();
|
||||
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
|
||||
let plan = crate::data_usage_define::DataUsageScanPlanDigest([31; 32]);
|
||||
let identity = crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::from_u128(31),
|
||||
set_layout: crate::data_usage_define::DataUsageScanPlanDigest([32; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
assert_eq!(
|
||||
initial.prepare_bucket_checkpoint("bucket", 1, 0, source, plan, identity),
|
||||
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
initial.info.skip_healing = true;
|
||||
initial.replace("bucket", "", DataUsageEntry::default());
|
||||
let initial = seed_flat_bucket(&disk_root, request.objects).await;
|
||||
tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
|
||||
.await
|
||||
.expect("persist initial cache");
|
||||
@@ -137,10 +156,11 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
&parent,
|
||||
crate::scanner_budget::ScannerCycleBudgetConfig {
|
||||
max_objects: Some(request.raw_entry_budget),
|
||||
max_entries: Some(request.raw_entry_budget),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let _observation_guard = install_raw_entry_budget(disk.path(), request.raw_entry_budget);
|
||||
let _observation_guard = install_raw_entry_observer(disk.path());
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user