Compare commits

..

26 Commits

Author SHA1 Message Date
overtrue 6d3b882aa1 Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-30 15:28:30 +08:00
overtrue e399112044 ci: isolate large pagination fixture in full e2e 2026-10-30 15:28:20 +08:00
overtrue 039a728213 Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-30 13:55:44 +08:00
overtrue a1ee666c4f fix(lock): serialize guard release with force unlock 2026-10-30 13:54:07 +08:00
overtrue 9518a5d1e2 fix(lock): reject force-released acquisitions before registration 2026-10-30 11:10:46 +08:00
overtrue 2bcc4895ae test(storage): wait for stable internal fixture snapshots 2026-10-30 10:53:14 +08:00
overtrue f36154a8aa fix(lock): serialize force unlock with tail conversion 2026-10-30 09:35:29 +08:00
overtrue 21cb36c8be Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-30 09:27:10 +08:00
overtrue e76648b5e3 fix(lock): allow reads while committed rename tails drain 2026-10-30 09:27:05 +08:00
overtrue 5cc296314e test(heal): wait for metadata write tail 2026-10-29 22:40:38 +08:00
overtrue 01d43729aa fix(heal): retain deep scan retry for read repair 2026-10-29 21:30:41 +08:00
overtrue 0a32ad9109 Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-29 21:16:34 +08:00
overtrue d0b26195c2 fix(heal): allow reads during read repair reconstruction 2026-10-29 21:16:34 +08:00
overtrue 7a740fe5e7 fix(lock): retain quorum error on final timeout 2026-10-29 09:13:54 +08:00
overtrue 2aaef5451d Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-29 09:04:36 +08:00
overtrue a213ce597e chore: merge main into heal lock fixes 2026-10-29 08:22:08 +08:00
overtrue c749cf9da5 chore: merge main into heal lock fixes 2026-10-29 06:50:01 +08:00
overtrue 1ab0612c9c chore: merge main into heal lock fixes 2026-10-29 01:55:22 +08:00
overtrue 6538f5ca36 test(heal): isolate multi-pool crash recovery 2026-10-28 21:57:29 +08:00
overtrue 7040cb65c1 Merge remote-tracking branch 'origin/main' into overtrue/fix-heal-lock-ci 2026-10-28 21:55:29 +08:00
overtrue 5427bd8f84 Merge branch 'main' of https://github.com/rustfs/rustfs into overtrue/fix-heal-lock-ci 2026-10-28 20:49:27 +08:00
overtrue 3c2b54aeb5 fix(lock): retain quorum failures across deadline retries 2026-10-28 07:53:19 +08:00
overtrue 0de223d8e6 test(lock): use virtual time for retry budget 2026-10-28 06:56:33 +08:00
overtrue 351bd12031 fix(lock): use remaining acquire budget for retries 2026-10-28 06:55:04 +08:00
overtrue a2381c8421 test(heal): isolate the direct repair fallback 2026-10-28 06:33:30 +08:00
overtrue 5eee702e88 test(heal): isolate durable repair during root recovery 2026-10-28 06:31:40 +08:00
372 changed files with 4692 additions and 51919 deletions
-1
View File
@@ -18,7 +18,6 @@ These implementation, style, and quality rules apply to all contributions, regar
- Do not include agent, tool, contributor, account, or organization names in branch names.
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
+1 -1
View File
@@ -1 +1 @@
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
+3 -3
View File
@@ -970,10 +970,10 @@ threads-required = "num-test-threads"
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
test-group = 'e2e-vault'
# Keep the storage-heavy smoke cases isolated in the full suite too. They
# start multiple server fixtures and issue many sequential S3 operations.
# Keep storage-heavy cases isolated in the full suite too. The large pagination
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
[[profile.e2e-full.overrides]]
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
threads-required = "num-test-threads"
# Fixed, small contract for every executable server change. These existing
-77
View File
@@ -1,77 +0,0 @@
{
"schema_version": 1,
"plan_id": "rustfs-release-validation",
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
"release_types": ["preview", "prerelease", "stable"],
"targets": [
{"os": "linux", "arch": "x86_64", "abi": "musl"},
{"os": "linux", "arch": "aarch64", "abi": "musl"},
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
{"os": "macos", "arch": "aarch64", "abi": "none"},
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
],
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
"features": ["pyroscope"],
"trusted_workflows": [
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
],
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
"lanes": [
{
"lane": "identity",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
"timeout_seconds": 1800,
"environment_capabilities": ["process-exec"],
"cases": ["artifact-identity", "startup"],
"applies_to": {
"release_types": ["preview", "prerelease", "stable"],
"platforms": "all"
},
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
},
{
"lane": "upgrade",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
"timeout_seconds": 5400,
"environment_capabilities": ["process-exec", "previous-release-artifact"],
"cases": [
"direct-upgrade",
"mixed-version-upgrade",
"bucket-config-upgrade",
"bucket-config-rollback",
"multipart-layout-upgrade",
"multipart-layout-baseline"
],
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
"applies_to": {
"release_types": ["prerelease", "stable"],
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
},
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.1",
"role": "N-1",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
},
{
"version": "1.0.0-rc.5",
"role": "N-2",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
}
}
+5 -311
View File
@@ -814,18 +814,13 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Determine upload path based on build type. Release and prerelease
# builds only stage their bytes here: the official
# artifacts/rustfs/release/ path is written solely by the
# certification-gated promote-release-artifacts job, from these same
# staged objects. Development builds keep the uncertified dev channel
# and never touch official or latest channels.
# Determine upload path based on build type
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
echo "📤 Uploading development build to R2 dev directory"
else
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
echo "📤 Staging release build for certification-gated promotion"
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
echo "📤 Uploading release build to R2 release directory"
fi
# Upload all rustfs zip files to R2 using S3-compatible API
@@ -874,221 +869,6 @@ jobs:
echo "This run does not trigger Docker publishing."
fi
# Aggregate the parent candidate batch. This job always collects results and
# retains its evidence, including on failure, and every publication job
# below requires its success. Missing, stale, substituted or non-PASS
# evidence fails closed here: only the trusted verification sources
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
# it, so a PR-uploaded JSON set never gains release authority.
release-certification:
name: Release Certification
needs: [ build-check, build-rustfs ]
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download all build artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Write candidate and collect trusted validation results
env:
# Cross-repo read of the trusted verification run. Falls back to the
# workflow token, which cannot read rustfs/s3chaos; a missing token
# therefore fail-closes instead of certifying.
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
PRODUCT_COMMIT: ${{ github.sha }}
VERSION: ${{ needs.build-check.outputs.version }}
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
shell: bash
run: |
set -euo pipefail
mkdir -p results release-gate-output
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
import hashlib, json, os, sys, zipfile
plan_sha256 = sys.argv[1]
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
if profile is None:
raise SystemExit("release certification only covers preview, prerelease and stable builds")
def sha256(path):
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
deliverables = []
for name in sorted(os.listdir("artifacts")):
if not name.startswith("rustfs-") or not name.endswith(".zip"):
continue
path = os.path.join("artifacts", name)
members = zipfile.ZipFile(path).namelist()
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
with zipfile.ZipFile(path) as archive:
binary_bytes = archive.read(binary_name)
import tempfile
with tempfile.NamedTemporaryFile() as handle:
handle.write(binary_bytes)
handle.flush()
binary_sha = sha256(handle.name)
deliverables.append({
"kind": "zip",
"name": name,
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
"sha256": sha256(path),
"size": os.path.getsize(path),
"binary_sha256": binary_sha,
})
if not deliverables:
raise SystemExit("no built deliverables found to certify")
candidate = {
"schema_version": 1,
"product_commit": os.environ["PRODUCT_COMMIT"],
"version": os.environ["VERSION"],
"release_profile": profile,
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
"plan_sha256": plan_sha256,
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
"deliverables": deliverables,
}
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
print()
PY
# Trusted verification evidence only: the release-gate envelopes of
# rustfs/s3chaos for this exact product commit. Absence blocks below.
TRUSTED_REPO="rustfs/s3chaos"
TRUSTED_WORKFLOW="release-gate.yml"
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
if [[ -z "$TRUSTED_RUN" ]]; then
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
else
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
fi
python3 scripts/release/release_gate.py validate \
--plan .config/release-validation-plan.json \
--candidate candidate.json \
--results results \
--output release-gate-output
- name: Upload certification evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-certification
path: |
candidate.json
results/
release-gate-output/
if-no-files-found: warn
retention-days: 30
# The only writer of the official R2 release path. Promotion re-checks the
# exact bytes about to be published against the passed certification list
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
promote-release-artifacts:
name: Promote Release Artifacts
needs: [ build-check, build-rustfs, release-certification ]
# Target-channel lock for the official R2 release path.
concurrency:
group: release-channel-r2-release
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download build and certification artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged release artifacts
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$name"
done
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
staged="$(mktemp)"
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged object ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
echo "✅ Promoted ${name} (${built_sha})"
done
# Create GitHub Release for every valid release tag, including previews
create-release:
name: Create GitHub Release
@@ -1248,10 +1028,6 @@ jobs:
update-latest-version:
name: Update Latest Version
needs: [ build-check, publish-release ]
# Target-channel lock for the latest.json version pointer.
concurrency:
group: release-channel-oss-latest
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -1292,45 +1068,6 @@ jobs:
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
chmod +x "$OSSUTIL_BIN"
# Candidate/pointer ordering: an older candidate must never overwrite
# a newer pointer, even if its promotion job finishes later.
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
import json
import re
import sys
def key(version):
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
if not match:
raise SystemExit(3)
core = tuple(int(part) for part in match.groups()[:3])
prerelease = match.group(4)
if prerelease is None:
return core + (1,)
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
try:
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
proposed = sys.argv[2]
if current and key(current) > key(proposed):
raise SystemExit(1)
except SystemExit:
raise
except (OSError, ValueError, TypeError):
raise SystemExit(3)
PY
guard_status=$?
if [[ "$guard_status" -eq 1 ]]; then
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
exit 0
elif [[ "$guard_status" -ne 0 ]]; then
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
exit 1
fi
fi
# Create latest.json
cat > latest.json << EOF
{
@@ -1347,59 +1084,16 @@ jobs:
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
# Publish release (remove draft status). Requires the release-certification
# aggregate: without a complete legal success there is no public write.
# Publish release (remove draft status)
publish-release:
name: Publish Release
needs: [ build-check, create-release, upload-release-assets, release-certification ]
# Target-channel lock: concurrent releases serialize on this channel so an
# older candidate cannot interleave its publish with a newer one.
concurrency:
group: release-channel-github-release
cancel-in-progress: false
needs: [ build-check, create-release, upload-release-assets ]
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: write
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify release assets against certification
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
TAG="${{ needs.build-check.outputs.version }}"
mkdir -p "$RUNNER_TEMP/promotion-assets"
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
found=0
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
[[ -f "$zip_file" ]] || continue
found=1
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$(basename "$zip_file")"
done
if [[ "$found" -ne 1 ]]; then
echo "No release zip assets to verify against the certification" >&2
exit 1
fi
- name: Publish release
env:
GH_TOKEN: ${{ github.token }}
+2 -2
View File
@@ -96,7 +96,7 @@ env:
RUST_BACKTRACE: 1
jobs:
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
# e2e-tests, e2e-full.
warm-ci-dev:
name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
if-no-files-found: error
# --all-targets covers the test binaries nextest builds, including
# e2e_test, which the workspace run excludes. The second build adds
# e2e_test, which test-and-lint's own run excludes. The second build adds
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
# and that no lint lane enables.
- name: Build ci-dev superset
+34 -36
View File
@@ -113,7 +113,7 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
@@ -160,7 +160,7 @@ jobs:
- name: Run shared quick checks
uses: ./.github/actions/quick-checks
workspace-lint-and-build:
test-and-lint:
name: Workspace Lint and Build
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
@@ -193,13 +193,13 @@ jobs:
- name: Prepare test evidence
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
{
echo "run_id=${GITHUB_RUN_ID}"
echo "job=${GITHUB_JOB}"
echo "runner=${RUNNER_NAME}"
echo "started_at=$(date --utc --iso-8601=seconds)"
} > artifacts/workspace/run-metadata.txt
} > artifacts/test-and-lint/run-metadata.txt
# Clippy gates every event, pull requests included. When it ran only after
# merge, a lint regression merged freely and then failed every main run
@@ -233,11 +233,11 @@ jobs:
- name: Run documentation tests
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
set +e
timeout --verbose --signal=TERM --kill-after=30s 15m \
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
2>&1 | tee artifacts/workspace/doctest.log
2>&1 | tee artifacts/test-and-lint/doctest.log
status=${PIPESTATUS[0]}
{
echo "command=cargo test --all --exclude e2e_test --doc"
@@ -246,7 +246,7 @@ jobs:
echo
echo "Remaining test-related processes:"
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
} > artifacts/workspace/doctest-diagnostics.txt
} > artifacts/test-and-lint/doctest-diagnostics.txt
exit "${status}"
- name: Upload test reports and diagnostics
@@ -255,7 +255,7 @@ jobs:
with:
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
path: |
artifacts/workspace
artifacts/test-and-lint
retention-days: 3
if-no-files-found: error
@@ -268,7 +268,7 @@ jobs:
workspace-tests:
name: Workspace tests
needs: [ workspace-lint-and-build, classify-changes ]
needs: [ test-and-lint, classify-changes ]
# The archive retains compile-time fixture paths and the large test set
# requires the same runner image until the hosted-runtime pilot passes.
runs-on: sm-standard-4
@@ -293,14 +293,14 @@ jobs:
- name: Extract and list workspace tests
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
> artifacts/workspace/core-test-listing.json
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
> artifacts/test-and-lint/core-test-listing.json
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
- name: Run nextest tests
run: |
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest
trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -309,7 +309,7 @@ jobs:
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
--status-level all --final-status-level all \
2>&1 | tee artifacts/workspace/nextest.log
2>&1 | tee artifacts/test-and-lint/nextest.log
status=${PIPESTATUS[0]}
{
echo "command=workspace archive execution"
@@ -321,13 +321,13 @@ jobs:
echo
echo "Kernel OOM / kill events:"
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
} > artifacts/workspace/nextest-diagnostics.txt
} > artifacts/test-and-lint/nextest-diagnostics.txt
exit "${status}"
- name: Verify workspace execution completeness
run: |
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
- name: Run ignored ILM integration tests serially
if: needs.classify-changes.outputs.mode == 'full'
@@ -358,7 +358,7 @@ jobs:
name: workspace-test-evidence-${{ github.run_attempt }}
path: |
target/nextest/ci/junit.xml
artifacts/workspace
artifacts/test-and-lint
artifacts/ilm-integration
retention-days: 3
if-no-files-found: error
@@ -368,7 +368,7 @@ jobs:
if: needs.classify-changes.outputs.rehearse_shards == 'true'
run: |
python3 scripts/ci_test_shards.py prepare \
--listing artifacts/workspace/core-test-listing.json \
--listing artifacts/test-and-lint/core-test-listing.json \
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
- name: Upload same-run shard plan
if: needs.classify-changes.outputs.rehearse_shards == 'true'
@@ -449,7 +449,7 @@ jobs:
connect-feature-boundaries:
name: Connect Feature Boundaries
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
runs-on: sm-standard-4
timeout-minutes: 90
@@ -499,13 +499,11 @@ jobs:
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
# Specialized boundary, hardware and alternate-feature lanes run only on
# the nightly schedule and manual dispatch. They do not gate a merge: the
# nightly run reports a regression within a day, and repeating these builds
# on every PR revision and merge group bought no earlier signal worth the
# runner occupancy. Workspace lint and tests, the core contract, the debug
# build, E2E smoke and the S3 lanes remain the pull-request gate.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
# Alternate feature configurations run on the nightly schedule, merge
# groups and manual dispatch. The default workspace remains the PR gate and
# the per-push main baseline: repeating four feature builds on every merge
# bought nothing the nightly run does not report within a day.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
runs-on: sm-standard-4
timeout-minutes: 90
@@ -542,12 +540,12 @@ jobs:
# --profile ci so the quarantine list (and its junit flaky markers)
# covers this leg too; the default profile is the local no-retry
# profile and silently ignored quarantined flakes here (rustfs#6703).
mkdir -p artifacts/workspace
mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest-rio-v2
trap './scripts/ci/resource_sampler.sh stop' EXIT
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
2>&1 | tee artifacts/workspace/nextest.log
2>&1 | tee artifacts/test-and-lint/nextest.log
cargo test -p rustfs --doc --features rio-v2
- name: Upload rio-v2 test reports and diagnostics
@@ -557,14 +555,14 @@ jobs:
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
path: |
target/nextest/ci/junit.xml
artifacts/workspace
artifacts/test-and-lint
retention-days: 3
if-no-files-found: error
test-and-lint-protocols:
name: "Test and Lint (${{ matrix.features.name }})"
# Same cadence as test-and-lint-rio-v2 above.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
needs: [ quick-checks, classify-changes ]
runs-on: sm-standard-4
timeout-minutes: 90
@@ -633,7 +631,7 @@ jobs:
path: |
target/nextest/ci/junit.xml
artifacts/protocol-tests
artifacts/workspace
artifacts/test-and-lint
retention-days: 3
if-no-files-found: warn
@@ -785,7 +783,7 @@ jobs:
# job had neither, so each closed/merged PR really ran the whole io_uring
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
# 30662728539) and kept the cancellation run in progress for minutes.
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
# a container, applies no seccomp filter that would block io_uring_setup — so
@@ -980,7 +978,7 @@ jobs:
path: |
${{ runner.temp }}/rustfs-e2e-smoke-logs/
${{ runner.temp }}/rustfs-e2e-smoke-list.json
artifacts/workspace
artifacts/test-and-lint
if-no-files-found: warn
- name: Upload e2e smoke JUnit report
@@ -1391,7 +1389,7 @@ jobs:
- classify-changes
- typos
- quick-checks
- workspace-lint-and-build
- test-and-lint
- workspace-tests
- workspace-shards
- workspace-shard-evidence
@@ -1427,7 +1425,7 @@ jobs:
- required-checks
- typos
- quick-checks
- workspace-lint-and-build
- test-and-lint
- workspace-tests
- core-contract
- test-and-lint-rio-v2
+2 -182
View File
@@ -329,75 +329,13 @@ jobs:
echo " - Create latest: $create_latest"
echo " - Source ref: $source_ref"
# Consume the trusted release certification for the exact version this
# workflow wraps and verify the certified input digests. A manual dispatch
# without that certification fails here and pushes no official tag; only
# results from the trusted verification sources carry release authority.
release-certification:
name: Release Certification
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
ref: ${{ needs.build-check.outputs.source_ref }}
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$run_id" ]]; then
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
exit 1
fi
if ! gh run download "$run_id" -n release-certification -D certification; then
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Input digest binding: every zip this image wraps must be the
# certified byte-for-byte content, never a mutable latest pointer.
while IFS=$'\t' read -r name location expected; do
echo "Verifying certified input ${name}"
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
if [[ "$actual" != "$expected" ]]; then
echo "Certified input ${name} does not match its recorded digest" >&2
exit 1
fi
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
# Build multi-arch Docker images
# Strategy: Build images using pre-built binaries from dl.rustfs.com
# Supports both release and dev channel binaries based on build context
# Only runs when should_build is true (which includes workflow success check)
build-docker:
name: Build Docker Images
needs: [ build-check, release-certification ]
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -505,14 +443,6 @@ jobs:
# Output tags
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
# Staging identity for this run's promotion: official tags are only
# created after the staged image verifies against the certification,
# and they point at this same digest (no rebuild-after-verify).
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
# Generate labels
LABELS="org.opencontainers.image.title=RustFS"
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -537,9 +467,7 @@ jobs:
file: ${{ matrix.file }}
platforms: ${{ env.DOCKER_PLATFORMS }}
push: ${{ needs.build-check.outputs.should_push == 'true' }}
# Push to staging tags only: official tags are created by the
# certification-gated promotion step below from this same digest.
tags: ${{ steps.meta.outputs.staging_tags }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# No layer cache. This build compiles nothing — it downloads a
# release zip and runs apk/apt — so the cache could only save the
@@ -565,114 +493,6 @@ jobs:
no-cache: false
pull: true
# A later-built wrapper: the staged image must complete this platform's
# own binary identity verification before any official tag is written.
# It may not inherit an unverified package's conclusion.
- name: Verify staged image against certification
if: needs.build-check.outputs.should_push == 'true'
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
case "${{ matrix.variant }}" in
musl) libc="musl" ;;
glibc) libc="gnu" ;;
esac
for platform in linux/amd64 linux/arm64; do
case "$platform" in
linux/amd64) arch_sub="x86_64" ;;
linux/arm64) arch_sub="aarch64" ;;
esac
docker pull --platform "$platform" "$STAGING_REF"
container="$(docker create --platform "$platform" "$STAGING_REF")"
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
docker rm "$container" >/dev/null
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/staged-rustfs" \
--artifact-role binary
done
- name: Promote staged image to official tags
if: needs.build-check.outputs.should_push == 'true'
shell: bash
run: |
set -euo pipefail
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
for tag in "${official_tags[@]}"; do
docker buildx imagetools create -t "$tag" "$STAGING_REF"
echo "Promoted $tag -> $STAGING_REF"
done
# Downstream consumers (Helm) verify this record instead of inheriting
# an unverified wrapper's conclusion.
- name: Record image promotion
if: needs.build-check.outputs.should_push == 'true'
shell: bash
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
jq -n \
--arg version "$VERSION" \
--arg digest "$digest" \
--arg staging_ref "$STAGING_REF" \
--arg candidate_sha256 "$candidate_sha" \
--arg tags "${{ steps.meta.outputs.tags }}" \
--slurpfile candidate certification/candidate.json \
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
> image-promotion-record.json
- name: Upload image promotion record
if: needs.build-check.outputs.should_push == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: image-promotion-record${{ matrix.suffix }}
path: image-promotion-record.json
if-no-files-found: error
retention-days: 30
# Note: Manifest creation is no longer needed as we only build one variant
# Multi-arch manifests are automatically created by docker/build-push-action
-93
View File
@@ -30,102 +30,9 @@ permissions:
contents: read
jobs:
# Consume the same trusted release certification as every other publication
# exit and verify the image promotion record of the Docker workflow. The
# chart wraps no binary of its own, so it may only reference an image whose
# platform verification already passed; it must not inherit an unverified
# package's conclusion. A run without that evidence fails closed here and
# never reaches the chart repository.
release-certification:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
outputs:
version: ${{ steps.version.outputs.raw_tag }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Normalize release version
id: version
env:
RAW_INPUT: ${{ github.event.inputs.version }}
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
RAW="$RAW_INPUT"
else
RAW="$RAW_BRANCH"
fi
case "$RAW" in
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
*) RAW_TAG="$RAW" ;;
esac
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
- name: Download and verify release certification and image record
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification image-record
build_run="$TRIGGERING_RUN_ID"
if [[ -z "$build_run" ]]; then
build_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$build_run" ]]; then
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
exit 1
fi
if ! gh run download "$build_run" -n release-certification -D certification; then
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
docker_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
exit 1
fi
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
if [[ "$record_sha" != "$candidate_sha" ]]; then
echo "Image promotion record does not belong to this certification" >&2
exit 1
fi
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
image-record/image-promotion-record.json)"
if [[ "$overlap" -ne 0 ]]; then
echo "Image promotion record carries binary digests outside the certification" >&2
exit 1
fi
build-helm-package:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ release-certification ]
if: |
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
(
+14 -210
View File
@@ -225,57 +225,10 @@ jobs:
echo " Build run ID: $BUILD_RUN_ID"
echo " Build run number: $RUN_NUMBER"
# Consume the trusted release certification of the resolved build run and
# verify the certified input digests before any package is wrapped. A run
# without that certification fails closed here; packages built later must
# complete their own verification and cannot inherit an unverified
# conclusion.
release-certification:
name: Release Certification
needs: resolve
if: needs.resolve.result == 'success'
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
if [[ "$BUILD_TYPE" == "development" ]]; then
# Development artifacts are explicitly uncertified and never touch
# official or latest channels; only the staging/dev paths remain.
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
> certification/development-uncertified.json
exit 0
fi
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Build DEB and RPM packages for each architecture and libc variant
package:
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
needs: [ resolve, release-certification ]
needs: resolve
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
strategy:
@@ -541,55 +494,6 @@ jobs:
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
echo "✅ RPM built: $RPM_FILE"
- name: Download certification evidence
if: needs.resolve.outputs.build_type != 'development'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
# Later-built wrapper verification: the binary inside each package must
# match the certified binary digest for this platform before anything is
# uploaded, and the package itself stays on staging until promotion.
- name: Verify wrapped binary against certification
if: needs.resolve.outputs.build_type != 'development'
shell: bash
env:
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
ARTIFACT_NAME: ${{ matrix.artifact_name }}
run: |
set -euo pipefail
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" ]]; then
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
exit 1
fi
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y cpio
fi
for package_file in "$DEB_FILE" "$RPM_FILE"; do
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
done
- name: Upload packages to artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
@@ -627,14 +531,10 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Build-phase writes stay on staging for release types: the official
# packages/release and packages/latest paths are written only by the
# certification-gated promote-packages job, from these same staged
# objects. Development packages keep the uncertified dev channel.
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PREFIX="artifacts/rustfs/packages/dev"
else
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
R2_PREFIX="artifacts/rustfs/packages/release"
fi
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
@@ -649,6 +549,18 @@ jobs:
echo "✅ Upload complete"
# Also upload as latest for release/prerelease
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
for f in "$DEB_FILE" "$RPM_FILE"; do
if [[ -n "$f" && -f "$f" ]]; then
echo "Uploading latest: $(basename "$f")"
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
fi
done
echo "✅ Latest packages updated"
fi
- name: Upload packages to GitHub Release
if: needs.resolve.outputs.tag != ''
env:
@@ -702,114 +614,6 @@ jobs:
echo "✅ GitHub release assets updated"
# The only writer of the official packages/release and packages/latest R2
# paths. Promotion re-checks each package's wrapped binary against the
# passed certification list and copies the staged objects byte-for-byte:
# no rebuild-after-verify, no mutable latest resolution downstream.
promote-packages:
name: Promote Packages
needs: [ resolve, package, release-certification ]
# Target-channel lock for the official package paths.
concurrency:
group: release-channel-r2-packages
cancel-in-progress: false
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download packages and certification
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./packages
pattern: packages-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged packages
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get install -y cpio
fi
promoted=0
for package_file in packages/*.deb packages/*.rpm; do
[[ -f "$package_file" ]] || continue
name="$(basename "$package_file")"
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
staged="$(mktemp)"
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged package ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
promoted=$((promoted + 1))
echo "✅ Promoted ${name} (${built_sha})"
done
if [[ "$promoted" -eq 0 ]]; then
echo "No staged packages found to promote" >&2
exit 1
fi
# Summary
summary:
name: Summary
Generated
+45 -56
View File
@@ -1321,9 +1321,9 @@ dependencies = [
[[package]]
name = "aws-smithy-schema"
version = "1.1.2"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9543fc722ed4865ab29fb66bd7ed0bef7b029d265ba8c587144511e30006d431"
checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
@@ -4198,9 +4198,9 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "encoding_rs"
version = "0.8.44"
version = "0.8.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5"
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
dependencies = [
"cfg-if",
"core_detect",
@@ -5277,9 +5277,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.30.2"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e"
checksum = "502dc37a7c218b9b1d3a0ccfb904da81d10d6dda0e899c722c503092d55576ea"
dependencies = [
"arc-swap",
"async-channel",
@@ -5312,15 +5312,15 @@ dependencies = [
[[package]]
name = "hotpath-drain"
version = "0.30.2"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0faae1a9e1fd8ce5debaaaf452dcbe833fc8a8a4f3779d2ec8828b116c3c16a"
checksum = "0ea17d04c231b90b1f68a91f6f80fb0762c5dea07c611dce9ae157b56a50ff4c"
[[package]]
name = "hotpath-macros"
version = "0.30.2"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58"
checksum = "a76e0af911930fc97abc766b393b48cdca77fea9f5f6b388cf2123a6d9c30deb"
dependencies = [
"proc-macro2",
"quote",
@@ -5899,9 +5899,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.107"
version = "0.3.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
dependencies = [
"cfg-if",
"futures-util",
@@ -6094,9 +6094,9 @@ dependencies = [
[[package]]
name = "libbz2-rs-sys"
version = "0.2.6"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32f454fd6831a29c0e827ed075bca2a15be201bd57f4168f0fd1041124eab17e"
checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
[[package]]
name = "libc"
@@ -8305,9 +8305,9 @@ dependencies = [
[[package]]
name = "pulldown-cmark"
version = "0.13.6"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b"
checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb"
dependencies = [
"bitflags 2.13.2",
"memchr",
@@ -8823,9 +8823,9 @@ dependencies = [
[[package]]
name = "redis"
version = "1.8.1"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db7f3d75b5a3301746bff39a767bf4c5c42506b08513527308b3875bfb64a031"
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
dependencies = [
"arc-swap",
"arcstr",
@@ -10617,9 +10617,6 @@ dependencies = [
"hotpath",
"http 1.5.0",
"metrics",
"opentelemetry-proto",
"opentelemetry_sdk",
"prost 0.14.4",
"rand 0.10.3",
"rmp-serde",
"rustfs-common",
@@ -10633,7 +10630,6 @@ dependencies = [
"rustfs-heal-contracts",
"rustfs-lifecycle",
"rustfs-lock",
"rustfs-obs",
"rustfs-s3-types",
"rustfs-scanner-metrics",
"rustfs-storage-api",
@@ -10848,7 +10844,8 @@ dependencies = [
[[package]]
name = "rustfs-uring"
version = "0.3.0"
source = "git+https://github.com/rustfs/uring.git?rev=eb8fc160e71d804427de1e905f5c226d580f71bf#eb8fc160e71d804427de1e905f5c226d580f71bf"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7deda770b7981ee3bfaa202ecd18625fc794c962b8d58cd4f436952b7e66dcf"
dependencies = [
"io-uring",
"libc",
@@ -12217,9 +12214,9 @@ dependencies = [
[[package]]
name = "sysinfo"
version = "0.40.1"
version = "0.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e8a3edbe0394ce115f3917a00349dd242be09eb5a091ed509eaf67b55cfb8a8"
checksum = "110e37d98880fd804cd47509b90a8444f7aaaab655ae625e83cd442a8bc2ebf2"
dependencies = [
"libc",
"memchr",
@@ -12710,9 +12707,9 @@ checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
[[package]]
name = "tonic"
version = "0.14.9"
version = "0.14.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b"
checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff"
dependencies = [
"async-trait",
"axum",
@@ -12743,9 +12740,9 @@ dependencies = [
[[package]]
name = "tonic-build"
version = "0.14.9"
version = "0.14.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4"
checksum = "2c07bd4e9f99e2a3ed8bebad596ae02a94e53745b68acc6900e22c3475c8bc44"
dependencies = [
"prettyplease 0.2.37",
"proc-macro2",
@@ -12755,9 +12752,9 @@ dependencies = [
[[package]]
name = "tonic-prost"
version = "0.14.9"
version = "0.14.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484"
checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6"
dependencies = [
"bytes",
"prost 0.14.4",
@@ -12766,9 +12763,9 @@ dependencies = [
[[package]]
name = "tonic-prost-build"
version = "0.14.9"
version = "0.14.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ea216ddc448e08e7839ab09e20e60b06b6a1c32745ccb212c0ebd161c199f1b"
checksum = "4a05d8957fac6b3542999796cb4449d45da790751962f974a457e5e00c43d78c"
dependencies = [
"prettyplease 0.2.37",
"proc-macro2",
@@ -13161,12 +13158,9 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf8_iter"
version = "1.1.0"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0"
dependencies = [
"utf_types",
]
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
@@ -13174,12 +13168,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "utf_types"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6"
[[package]]
name = "uuid"
version = "1.29.0"
@@ -13294,11 +13282,12 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.130"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
@@ -13306,9 +13295,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.80"
version = "0.4.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501"
checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e"
dependencies = [
"js-sys",
"tokio",
@@ -13317,9 +13306,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.130"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -13327,9 +13316,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.130"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -13340,9 +13329,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.130"
version = "0.2.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
dependencies = [
"unicode-ident",
]
@@ -13362,9 +13351,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.107"
version = "0.3.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88"
checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4"
dependencies = [
"js-sys",
"wasm-bindgen",
+6 -6
View File
@@ -175,9 +175,9 @@ tokio-rustls = { default-features = false, version = "0.26.6" }
tokio-stream = { version = "0.1.19" }
tokio-test = "0.4.6"
tokio-util = { version = "0.7.20" }
tonic = { version = "0.14.9" }
tonic-prost = { version = "0.14.9" }
tonic-prost-build = { version = "0.14.9" }
tonic = { version = "0.14.8" }
tonic-prost = { version = "0.14.8" }
tonic-prost-build = { version = "0.14.8" }
tower = { version = "0.5.3" }
tower-http = { version = "0.7.1" }
h3 = "0.0.8"
@@ -319,7 +319,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
reed-solomon-simd = "3.1.0"
regex = { version = "1.13.1" }
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
redis = { version = "1.8.1" }
redis = { version = "1.8.0" }
rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.6" }
rust-embed = { version = "8.13.0" }
@@ -334,7 +334,7 @@ compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.4.0" }
strum = { version = "0.28.0" }
sysinfo = "0.40.1"
sysinfo = "0.40.0"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.0"
@@ -386,7 +386,7 @@ dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.30.2", default-features = false }
hotpath = { version = "0.30.1", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
+14 -83
View File
@@ -10,10 +10,6 @@ use std::time::Duration;
use tokio::sync::{Notify, Semaphore};
use tokio::time::Instant;
pub mod permit_set;
const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
/// A resource owner publishes a capacity; running units retain their permits
/// after a reduction. Lease slots are reserved once, with no per-unit Arc
/// allocation. Tokio wait queues may lazily initialize platform mutex storage
@@ -40,7 +36,6 @@ pub struct AdaptiveAdmission {
maximum: u32,
slots: Box<[PermitSlots]>,
next_chunk: AtomicUsize,
completed: AtomicU64,
}
struct PermitSlots {
@@ -66,12 +61,10 @@ pub struct AdaptivePermit {
impl Clone for AdaptivePermit {
fn clone(&self) -> Self {
// Subset carriers can retain slot references through a shared owner,
// so the slot count needs its own bound in addition to Arc's guard.
// Every slot reference owns a strong owner reference. Clone the Arc
// first, so its overflow protection also bounds the slot refcount.
let owner = Arc::clone(&self.owner);
if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
std::process::abort();
}
owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
Self { owner, slot: self.slot }
}
}
@@ -121,22 +114,11 @@ impl NodeAdmission {
/// Publish feedback with this owner's actual completed-work counter.
pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
self.update_with_probe(controller, observation, true)
}
/// Coordinate growth with other resource controllers in the same window.
/// Protective reductions never depend on permission to probe.
pub fn update_with_probe(
&self,
controller: &mut FeedbackController,
observation: Option<FeedbackObservation>,
allow_probe: bool,
) -> FeedbackDecision {
let observation = observation.map(|mut observation| {
observation.completed_units = self.completed();
observation
});
controller.update_with_probe(&self.capacity, observation, allow_probe)
controller.update(&self.capacity, observation)
}
/// Current leased scheduling cap and held async units.
@@ -267,31 +249,16 @@ impl FeedbackController {
/// Consume one sample and publish a three-second lease to the same gate.
/// If this function stops being called, admission expires independently.
pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
self.update_with_probe(gate, observation, true)
}
/// Allow at most the sampling owner's selected controller to probe upward.
pub fn update_with_probe(
&mut self,
gate: &AdaptiveAdmission,
observation: Option<FeedbackObservation>,
allow_probe: bool,
) -> FeedbackDecision {
let observation = observation.map(|mut observation| {
observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
observation
});
let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
let decision = self.decide(Instant::now(), observation);
gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
decision
}
fn decide_with_probe(
&mut self,
now: Instant,
observation: Option<FeedbackObservation>,
allow_probe: bool,
) -> FeedbackDecision {
fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
let Some(observation) = observation else {
self.previous = None;
self.stable_since = None;
@@ -336,8 +303,7 @@ impl FeedbackController {
FeedbackReason::NewBaseline
} else {
let since = *self.stable_since.get_or_insert(now);
if allow_probe
&& advanced
if advanced
&& now.saturating_duration_since(since) >= Duration::from_secs(8)
&& now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
&& self.limit < observation.max_concurrency
@@ -359,13 +325,6 @@ impl FeedbackController {
}
impl AdaptiveAdmission {
/// Successful async operations using this physical owner. Native clones
/// retain capacity but never count another completion.
pub fn completed(&self) -> Option<u64> {
let value = self.completed.load(Ordering::Acquire);
(value != u64::MAX).then_some(value)
}
/// Construct a bounded owner, with zero initial capacity allowed for unknown
/// input. The immutable maximum reserves at most 65,535 lease slots once;
/// acquiring or cloning a permit does not allocate a per-unit Arc.
@@ -392,7 +351,6 @@ impl AdaptiveAdmission {
maximum: maximum_u32,
slots,
next_chunk: AtomicUsize::new(0),
completed: AtomicU64::new(0),
})
}
@@ -508,21 +466,6 @@ impl AdaptiveAdmission {
self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
}
fn release_reference(&self, slot: usize) {
self.release_reference_with(slot, || {});
}
fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
return;
}
before_release();
// Publish the free slot before exposing another admission capacity.
self.release_slot(slot);
self.state.fetch_sub(1, Ordering::AcqRel);
self.changed.notify_one();
}
/// Wait for capacity. The future owns no permit until it returns; dropping
/// a pending future leaves capacity untouched. Wrap in the caller's existing
/// cancellation/deadline select before acquiring any storage commit locks.
@@ -550,7 +493,13 @@ impl AdaptiveAdmission {
impl Drop for AdaptivePermit {
fn drop(&mut self) {
self.owner.release_reference(self.slot);
if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
return;
}
// Publish the free slot before exposing another admission capacity.
self.owner.release_slot(self.slot);
self.owner.state.fetch_sub(1, Ordering::AcqRel);
self.owner.changed.notify_one();
}
}
@@ -912,24 +861,6 @@ mod tests {
}
}
#[tokio::test(start_paused = true)]
async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
let mut controller = FeedbackController::new();
controller.update_with_probe(&gate, Some(healthy(0)), false);
for completed in 1..=20 {
tokio::time::advance(Duration::from_secs(1)).await;
assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
}
tokio::time::advance(Duration::from_secs(1)).await;
assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
assert_eq!(gate.snapshot().limit, 2);
let mut busy = healthy(22);
busy.pressure = ResourcePressure::Busy;
assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
}
#[tokio::test(start_paused = true)]
async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
File diff suppressed because it is too large Load Diff
-12
View File
@@ -30,16 +30,6 @@ pub const COMMENT_KEY: &str = "comment";
pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
/// Global switch for enabling the notify module.
pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
/// Maximum audit entries per second for authorization denials of authenticated callers.
/// `0` disables denial auditing for that class of caller.
pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
/// Maximum audit entries per second for authorization denials of anonymous callers.
/// `0` disables denial auditing for that class of caller.
pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
/// Default denial audit rate for authenticated callers (entries per second).
pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
/// Default denial audit rate for anonymous callers (entries per second).
pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
/// Default global audit switch (disabled by default).
pub const DEFAULT_AUDIT_ENABLE: bool = false;
/// Default global notify switch (disabled by default).
@@ -338,7 +328,5 @@ mod tests {
fn test_global_audit_notify_switch_constants() {
assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
}
}
+53 -102
View File
@@ -24,14 +24,6 @@ use std::{
time::{Duration, SystemTime},
};
/// Copy an existing cache-model value without including surrounding work.
#[inline]
pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
#[cfg(feature = "hotpath-alloc")]
let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
source.to_owned()
}
/// Maximum amount a persisted `last_update` may lead the local wall clock before the
/// persisted timestamp is treated as untrustworthy.
///
@@ -350,7 +342,7 @@ impl UnknownTierStats {
self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
continue;
}
self.diagnostics.push(clone_cache_value(diagnostic));
self.diagnostics.push(diagnostic.clone());
}
}
if !self.diagnostics.is_empty() {
@@ -375,7 +367,7 @@ impl UnknownTierStats {
if !self.fits_add(other) {
return None;
}
let mut merged = clone_cache_value(self);
let mut merged = self.clone();
merged.merge(other);
Some(merged)
}
@@ -415,7 +407,7 @@ impl AllTierStats {
if st.is_empty() {
continue;
}
let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
let entry = self.tiers.entry(tier.clone()).or_default();
*entry = entry.add(st);
}
}
@@ -445,7 +437,7 @@ impl AllTierStats {
.tiers
.keys()
.filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
.map(clone_cache_value)
.cloned()
.collect();
for tier in retired_tiers {
if let Some(stats) = self.tiers.remove(&tier) {
@@ -1245,10 +1237,10 @@ impl DataUsageEntry {
}
pub fn add_child(&mut self, hash: &DataUsageHash) {
if self.children.contains(&clone_cache_value(&hash.0)) {
if self.children.contains(&hash.key()) {
return;
}
self.children.insert(clone_cache_value(&hash.0));
self.children.insert(hash.key());
}
pub fn merge(&mut self, other: &DataUsageEntry) {
@@ -1265,7 +1257,7 @@ impl DataUsageEntry {
s_rep.replica_size += o_rep.replica_size;
s_rep.replica_count += o_rep.replica_count;
for (arn, stat) in o_rep.targets.iter() {
let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
let st = s_rep.targets.entry(arn.clone()).or_default();
st.pending_size += stat.pending_size;
st.replicated_size += stat.replicated_size;
st.failed_size += stat.failed_size;
@@ -1459,7 +1451,7 @@ pub fn prefix_usage_in_cache(
} else {
format!("{bucket}/{prefix}")
};
let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
let entry = cache.get(&hash_path(&root).key())?.clone();
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
@@ -1602,38 +1594,38 @@ pub struct DataUsageCache {
impl DataUsageCache {
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
let hash = hash_path(path);
self.cache.insert(clone_cache_value(&hash.0), e);
self.cache.insert(hash.key(), e);
if !parent.is_empty() {
let phash = hash_path(parent);
let p = {
let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
let p = self.cache.entry(phash.key()).or_default();
p.add_child(&hash);
clone_cache_value(p)
p.clone()
};
self.cache.insert(clone_cache_value(&phash.0), p);
self.cache.insert(phash.key(), p);
}
}
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
self.cache.insert(hash.key(), e.clone());
if let Some(parent) = parent {
self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
self.cache.entry(parent.key()).or_default().add_child(hash);
}
}
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
self.cache.get(&hash_path(path).key()).cloned()
}
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
self.cache.entry(h.string()).or_default().children.clone()
}
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
let mut root = clone_cache_value(root);
for id in clone_cache_value(&root.children).iter() {
let mut root = root.clone();
for id in root.children.clone().iter() {
if let Some(e) = self.cache.get(id) {
let mut e = clone_cache_value(e);
let mut e = e.clone();
if !e.children.is_empty() {
e = self.flatten(&e);
}
@@ -1645,16 +1637,16 @@ impl DataUsageCache {
}
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
if let Some(e) = src.cache.get(&hash.string()) {
self.cache.insert(hash.key(), e.clone());
for ch in e.children.iter() {
if *ch == clone_cache_value(&hash.0) {
if *ch == hash.key() {
return;
}
self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
}
if let Some(parent) = parent {
let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
let p = self.cache.entry(parent.key()).or_default();
p.add_child(hash);
}
}
@@ -1662,14 +1654,14 @@ impl DataUsageCache {
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
let mut need_remove = Vec::new();
if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
if let Some(v) = self.cache.get(&hash.string()) {
for child in v.children.iter() {
need_remove.push(clone_cache_value(child));
need_remove.push(child.clone());
}
}
self.cache.remove(&clone_cache_value(&hash.0));
self.cache.remove(&hash.string());
need_remove.iter().for_each(|child| {
self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
self.delete_recursive(&DataUsageHash(child.to_string()));
});
}
@@ -1690,7 +1682,7 @@ impl DataUsageCache {
}
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
let want = clone_cache_value(&hash.0);
let want = hash.key();
if let Some(last_index) = want.rfind('/')
&& let Some(v) = self.find(&want[0..last_index])
&& v.children.contains(&want)
@@ -1701,7 +1693,7 @@ impl DataUsageCache {
for (k, v) in self.cache.iter() {
if v.children.contains(&want) {
let found = DataUsageHash(clone_cache_value(k));
let found = DataUsageHash(k.clone());
return Some(found);
}
}
@@ -1709,7 +1701,7 @@ impl DataUsageCache {
}
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
match self.cache.get(&clone_cache_value(&hash.0)) {
match self.cache.get(&hash.key()) {
Some(due) => due.compacted,
None => false,
}
@@ -1734,7 +1726,7 @@ impl DataUsageCache {
if self.cache.len() < limit {
return;
}
let top = clone_cache_value(&hash_path(&self.info.name).0);
let top = hash_path(&self.info.name).key();
let top_e = match self.find(&top) {
Some(e) => e,
None => return,
@@ -1761,7 +1753,7 @@ impl DataUsageCache {
}
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
let e = match self.cache.get(&clone_cache_value(&path.0)) {
let e = match self.cache.get(&path.key()) {
Some(e) => e,
None => return,
};
@@ -1771,13 +1763,13 @@ impl DataUsageCache {
}
if e.children.len() > limit && compact_self {
let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
flat.compacted = true;
self.delete_recursive(path);
self.replace_hashed(path, &None, &flat);
return;
}
let total = self.total_children_rec(&clone_cache_value(&path.0));
let total = self.total_children_rec(&path.key());
if total < limit {
return;
}
@@ -1790,12 +1782,12 @@ impl DataUsageCache {
let mut candidate_index = 0;
while remove > 0 && candidate_index < candidates.len() {
let e = &candidates[candidate_index];
let candidate = clone_cache_value(&e.path);
let candidate = e.path.clone();
if candidate == *path && !compact_self {
break;
}
let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
let removing = self.total_children_rec(&candidate.key());
let mut flat = match self.size_recursive(&candidate.key()) {
Some(flat) => flat,
None => {
candidate_index += 1;
@@ -1832,7 +1824,7 @@ impl DataUsageCache {
if o.root().is_none() {
return;
}
*self = clone_cache_value(o);
*self = o.clone();
return;
};
@@ -1845,8 +1837,7 @@ impl DataUsageCache {
}
existing_root.merge(&other_root);
self.cache
.insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
self.cache.insert(hash_path(&self.info.name).key(), existing_root);
let root_hash = self.root_hash();
for key in other_root.children.iter() {
@@ -1857,7 +1848,7 @@ impl DataUsageCache {
if let Some(existing) = self.cache.get_mut(key) {
existing.merge(&flat);
} else {
self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
}
}
}
@@ -1901,7 +1892,7 @@ impl DataUsageCache {
for (arn, stat) in rs.targets.iter() {
bui.replication_info.insert(
clone_cache_value(arn),
arn.clone(),
BucketTargetUsageInfo {
replication_pending_size: stat.pending_size,
replicated_size: stat.replicated_size,
@@ -1914,7 +1905,7 @@ impl DataUsageCache {
);
}
}
buckets_usage.insert(clone_cache_value(bucket_name), bui);
buckets_usage.insert(bucket_name.clone(), bui);
}
DataUsageInfo {
@@ -1946,20 +1937,20 @@ struct Inner {
}
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
let e = match data_usage_cache.cache.get(&path.key()) {
Some(e) => e,
None => return 0,
};
let mut objects = e.objects;
for ch in e.children.iter() {
objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
}
// Collect internal nodes (with children) as compaction candidates.
// Leaf nodes have no children to remove, so compacting them is a no-op.
if !e.children.is_empty() {
candidates.push(Inner {
objects,
path: clone_cache_value(path),
path: path.clone(),
});
}
objects
@@ -1967,7 +1958,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
for k in entry.children.iter() {
found.insert(clone_cache_value(k));
found.insert(k.to_string());
if let Some(ch) = duc.cache.get(k) {
mark(duc, ch, found);
}
@@ -2417,11 +2408,11 @@ impl SizeSummary {
}
for entry in &other.size_reconciliation {
self.record_size_reconciliation(clone_cache_value(entry));
self.record_size_reconciliation(entry.clone());
}
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
for scope in &other.reconciliation_scopes {
self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
self.record_reconciliation_scope(&scope.bucket, &scope.object);
}
}
@@ -2446,13 +2437,6 @@ impl SizeSummary {
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
/// merging summaries remains bounded and deterministic.
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
bucket: bucket.to_string(),
object: object.to_string(),
});
}
fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
if !self
.reconciliation_scopes
.iter()
@@ -2462,7 +2446,10 @@ impl SizeSummary {
self.size_reconciliation_truncated = true;
return;
}
self.reconciliation_scopes.push(build());
self.reconciliation_scopes.push(SizeReconciliationScope {
bucket: bucket.to_string(),
object: object.to_string(),
});
}
}
}
@@ -3417,42 +3404,6 @@ mod tests {
assert_eq!(summary1.versions, 15);
}
#[test]
fn test_reconciliation_scope_builds_only_after_admission() {
let mut summary = SizeSummary::new();
let builds = std::cell::Cell::new(0);
let build = || {
builds.set(builds.get() + 1);
SizeReconciliationScope {
bucket: "bucket".to_string(),
object: "object".to_string(),
}
};
summary.record_reconciliation_scope_with("bucket", "object", build);
assert_eq!(builds.get(), 1, "an admitted scope is copied once");
assert_eq!(summary.reconciliation_scopes.len(), 1);
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
assert_eq!(summary.reconciliation_scopes[0].object, "object");
summary.record_reconciliation_scope_with("bucket", "object", build);
assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
assert_eq!(summary.reconciliation_scopes.len(), 1);
assert!(!summary.size_reconciliation_truncated);
for index in 1..1024 {
summary.record_reconciliation_scope("bucket", &index.to_string());
}
summary.record_reconciliation_scope_with("bucket", "object", build);
assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
assert!(!summary.size_reconciliation_truncated);
summary.record_reconciliation_scope_with("bucket", "overflow", build);
assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
assert_eq!(summary.reconciliation_scopes.len(), 1024);
assert!(summary.size_reconciliation_truncated);
}
#[test]
fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
let mut hist = SizeHistogram::default();
-1
View File
@@ -25,7 +25,6 @@ workspace = true
[features]
default = []
scanner-peer-mixed-version = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
-19
View File
@@ -64,25 +64,6 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
### Scanner mixed-version gate
The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
```bash
python3 scripts/e2e_binary.py build --features e2e-test-hooks
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
cargo nextest run --profile e2e-distributed -p e2e_test \
--features scanner-peer-mixed-version --ignore-default-filter \
-E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
--test-threads 1 --retries 0 --no-tests=fail
```
The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
### `#[ignore]` semantics
Ignored tests are excluded from the default `cargo nextest run` pass because
@@ -16,193 +16,8 @@
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::config::http::HttpResponse;
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("NoSuchKey"),
"{error:?}"
);
}
#[tokio::test]
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
env.start_rustfs_server(vec![]).await.expect("start RustFS");
let client = env.create_s3_client();
let bucket = "conditional-delete-marker-proof";
create_versioned_bucket(&client, bucket).await;
client
.put_object()
.bucket(bucket)
.key("source")
.body(ByteStream::from_static(b"source"))
.send()
.await
.expect("write copy source");
for operation in ["put", "copy", "multipart"] {
let old = client
.put_object()
.bucket(bucket)
.key(operation)
.body(ByteStream::from_static(b"old"))
.send()
.await
.expect("write historical version");
let old_etag = old.e_tag().expect("historical ETag");
let marker = client
.delete_object()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("create delete marker");
let marker_id = marker.version_id().expect("delete marker version");
let (upload_id, completed) = if operation == "multipart" {
let upload = client
.create_multipart_upload()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("initiate multipart upload");
let upload_id = upload.upload_id().expect("upload ID").to_owned();
let part = client
.upload_part()
.bucket(bucket)
.key(operation)
.upload_id(&upload_id)
.part_number(1)
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect("upload part");
let completed = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.set_e_tag(part.e_tag().map(str::to_owned))
.build(),
)
.build();
(Some(upload_id), Some(completed))
} else {
(None, None)
};
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
match operation {
"put" => {
let err = client
.put_object()
.bucket(bucket)
.key(operation)
.if_match(etag)
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect_err("If-Match must reject marker");
assert_not_found(&err);
}
"copy" => {
let err = client
.copy_object()
.bucket(bucket)
.key(operation)
.copy_source(format!("{bucket}/source"))
.if_match(etag)
.send()
.await
.expect_err("If-Match copy must reject marker");
assert_not_found(&err);
}
_ => {
let err = client
.complete_multipart_upload()
.bucket(bucket)
.key(operation)
.upload_id(upload_id.as_deref().expect("upload ID"))
.multipart_upload(completed.clone().expect("completed parts"))
.if_match(etag)
.send()
.await
.expect_err("If-Match completion must reject marker");
assert_not_found(&err);
}
}
let versions = client
.list_object_versions()
.bucket(bucket)
.prefix(operation)
.send()
.await
.expect("list versions after rejected write");
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
assert_eq!(versions.delete_markers().len(), 1);
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
}
match operation {
"put" => {
client
.put_object()
.bucket(bucket)
.key(operation)
.if_none_match("*")
.body(ByteStream::from_static(b"new"))
.send()
.await
.expect("If-None-Match PUT over marker");
}
"copy" => {
client
.copy_object()
.bucket(bucket)
.key(operation)
.copy_source(format!("{bucket}/source"))
.if_none_match("*")
.send()
.await
.expect("If-None-Match copy over marker");
}
_ => {
client
.complete_multipart_upload()
.bucket(bucket)
.key(operation)
.upload_id(upload_id.as_deref().expect("upload ID"))
.multipart_upload(completed.expect("completed parts"))
.if_none_match("*")
.send()
.await
.expect("If-None-Match completion over marker");
}
}
let current = client
.get_object()
.bucket(bucket)
.key(operation)
.send()
.await
.expect("read successful conditional write")
.body
.collect()
.await
.expect("read object body")
.into_bytes();
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
}
}
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
async fn create_versioned_bucket(client: &Client, bucket: &str) {
client
-2
View File
@@ -39,8 +39,6 @@ mod s3_during_data_movement_test;
mod scanner_heal_topology_fixture;
mod scanner_heal_topology_test;
mod scanner_heal_topology_tier_test;
mod scanner_peer_bucket_fixture;
mod scanner_peer_bucket_test;
mod site_replication_test;
mod table_catalog_test;
mod upgrade_test;
@@ -18,7 +18,7 @@ use super::harness::{
};
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
use aws_sdk_s3::{
Client,
config::retry::RetryConfig,
@@ -38,16 +38,14 @@ const DEADLINE: Duration = Duration::from_secs(45);
async fn fixture(
drives_per_node: usize,
storage_class: &str,
lock_diagnostics: bool,
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
fixture_from_binary(drives_per_node, storage_class, None).await
}
async fn fixture_from_binary(
drives_per_node: usize,
storage_class: &str,
binary: Option<&std::path::Path>,
lock_diagnostics: bool,
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
let mut cluster =
RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
@@ -92,10 +90,6 @@ async fn fixture_from_binary(
] {
cluster.set_env(name, value);
}
if lock_diagnostics {
cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
}
let mut proxies = Vec::new();
if binary.is_none() {
for node in 0..4 {
@@ -369,120 +363,26 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
))
}
async fn with_paused_peers<T, E>(
async fn with_paused_peers<T: Send + 'static>(
proxies: &[RpcFaultProxy],
methods: &[&'static str],
limits: &[usize],
scope: &str,
request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
) -> TestResult<Result<T, SdkError<E>>>
where
T: Send + 'static,
E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
{
let deadline = tokio::time::Instant::now() + DEADLINE;
request: impl std::future::Future<Output = T> + Send + 'static,
) -> TestResult<T> {
let pauses: Vec<_> = proxies[1..=limits.len()]
.iter()
.zip(limits)
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
.collect::<Result<_, _>>()?;
let mut task = tokio::spawn(request);
let mut joined = None;
let mut deadline_expired = false;
let mut barrier_result = {
let faults = wait_for_faults(&pauses, methods);
tokio::pin!(faults);
match tokio::time::timeout_at(deadline, async {
tokio::select! {
result = &mut faults => result,
result = &mut task => {
joined = Some(result);
// A ready request is not evidence that every selected fault
// was reached. Poll the original barrier once for that proof.
match futures::poll!(faults.as_mut()) {
std::task::Poll::Ready(result) => result,
std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
}
}
}
})
.await
{
Ok(result) => result,
Err(error) => {
deadline_expired = true;
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
}
}
};
// Tokio's timeout polls an already-ready inner future before its timer.
// A late observation cannot certify the original absolute deadline.
deadline_expired |= tokio::time::Instant::now() >= deadline;
if deadline_expired {
barrier_result = Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"selected RPC fault proof observed after the absolute MPU deadline",
)
.into());
}
if barrier_result.is_err() {
for pause in &pauses {
pause.discard_pending();
}
}
let mut client_task_aborted = false;
let request_result = match joined {
Some(result) => result,
None => match tokio::time::timeout_at(deadline, &mut task).await {
Ok(result) => result,
Err(error) => {
deadline_expired = true;
for pause in &pauses {
pause.discard_pending();
}
barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
// This owns only the test's SDK request, not server/native work.
// Retire the client task without granting another operation budget.
client_task_aborted = true;
task.abort();
task.await
}
},
};
deadline_expired |= tokio::time::Instant::now() >= deadline;
if deadline_expired {
barrier_result = Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"SDK request result observed after the absolute MPU deadline",
)
.into());
}
let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
let task = tokio::spawn(request);
wait_for_faults(&pauses, methods).await?;
let result = tokio::time::timeout(DEADLINE, task).await??;
for pause in &pauses {
pause.discard_pending();
}
drop(pauses);
if barrier_result.is_err() {
let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
let response = sdk_error.and_then(SdkError::raw_response);
eprintln!(
"MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
sdk_error
.and_then(|error| error.as_service_error())
.and_then(|error| error.meta().code()),
response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
response.and_then(|response| response.headers().get("x-amz-request-id")),
);
}
match (barrier_result, request_result) {
// Preserve the actual SDK source instead of replacing an early service
// failure with an unrelated fault-wait timeout. The outer Err stays red.
(Err(_), Ok(Err(error))) => Err(Box::new(error)),
(Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
(Err(error), _) => Err(error),
(Ok(()), Ok(result)) => Ok(result),
(Ok(()), Err(error)) => Err(Box::new(error)),
}
Ok(result)
}
fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
@@ -1418,7 +1318,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
#[tokio::test]
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
@@ -1426,13 +1326,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
#[tokio::test]
async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
}
#[tokio::test]
async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
let storage: serde_json::Value = serde_json::from_str(&storage)?;
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
@@ -1446,7 +1346,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
#[tokio::test]
async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
let previous = super::upgrade_test::source_binary()?;
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
assert!(
proxies.is_empty(),
"the genuine previous fleet boots and seeds through its normal direct transport"
@@ -1649,7 +1549,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
capability_sha256: Option<[u8; 32]>,
abort: Option<AbortRef>,
}
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
let key = "partial-abort";
let committed = payload_for("committed-before-abort", 4096);
clients[0]
@@ -1865,7 +1765,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
#[tokio::test]
async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
for size in [0, 8 * 1024 * 1024] {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let key = "late-complete";
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
@@ -1992,7 +1892,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
for recreate_bucket in [false, true] {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let key = "late-part-transaction";
let previous = payload_for("committed-before-late-part", 4096);
clients[0]
@@ -2175,7 +2075,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
#[tokio::test]
async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let pauses: Vec<_> = proxies[1..]
.iter()
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
@@ -2198,7 +2098,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
use md5::{Digest as _, Md5};
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
for cancel_waiter in [false, true] {
for complete in [false, true] {
let key = format!("resend-{cancel_waiter}-{complete}");
@@ -2686,7 +2586,7 @@ async fn resume_at_rpc<T: Send + 'static>(
#[tokio::test]
async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
for locks in [false, true] {
let key = format!("phase-{locks}");
let client = clients[0].clone();
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+28 -456
View File
@@ -28,8 +28,7 @@ use hyper_util::{
use prost::Message as _;
use rustfs_protos::proto_gen::node_service::{
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
SettlePartTransactionResponse,
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
};
use std::{
collections::HashSet,
@@ -50,9 +49,6 @@ use tokio::{
type ProxyBody = BoxBody<Bytes, io::Error>;
type ProxyClient = Client<HttpConnector, ProxyBody>;
mod scoped_ack;
pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
#[derive(Clone, Copy, PartialEq, Eq)]
enum Disposition {
Hold,
@@ -125,8 +121,7 @@ impl RpcPause {
self.state.blocked.load(Ordering::SeqCst)
}
/// Selected RPCs with successful HTTP/gRPC response headers, excluding
/// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
/// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
pub(crate) fn forwarded(&self) -> usize {
self.state.forwarded.load(Ordering::SeqCst)
}
@@ -238,7 +233,6 @@ pub(crate) struct RpcFaultProxy {
address: SocketAddr,
slot: PauseSlot,
lock_slot: PauseSlot,
scoped_ack: scoped_ack::QueueSlot,
task: JoinHandle<()>,
late_requests: LateRequests,
}
@@ -251,8 +245,6 @@ impl RpcFaultProxy {
let active = slot.clone();
let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
let active_lock = lock_slot.clone();
let scoped_ack = scoped_ack::empty_slot();
let active_ack = scoped_ack.clone();
let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
let late = late_requests.clone();
let task = tokio::spawn(async move {
@@ -267,7 +259,6 @@ impl RpcFaultProxy {
let h2 = h2.clone();
let active = active.clone();
let active_lock = active_lock.clone();
let active_ack = active_ack.clone();
let late = late.clone();
connections.spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
@@ -281,7 +272,6 @@ impl RpcFaultProxy {
None
};
let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
let retain = pause.as_ref().is_some_and(|pause| {
pause.retain_after_disconnect
&& request.uri().path().strip_prefix("/node_service.NodeService/")
@@ -290,11 +280,11 @@ impl RpcFaultProxy {
let late = late.clone();
async move {
if !retain {
return forward_request(request, client, target, pause, ack).await;
return forward_request(request, client, target, pause).await;
}
// This opt-in fault owns the packet after client cancellation.
// Proxy teardown still aborts every retained request.
let task = tokio::spawn(forward_request(request, client, target, pause, ack));
let task = tokio::spawn(forward_request(request, client, target, pause));
{
let mut requests = late.lock().expect("late RPC ownership");
if let Some(requests) = requests.as_mut() {
@@ -319,7 +309,6 @@ impl RpcFaultProxy {
address,
slot,
lock_slot,
scoped_ack,
task,
late_requests,
})
@@ -329,11 +318,6 @@ impl RpcFaultProxy {
self.address
}
/// Retain only genuine scoped ACKs for one receiver/session and bucket set.
pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
}
/// Pause at most limit requests to the named unary methods; other streams
/// on the same HTTP/2 connection continue forwarding.
pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
@@ -485,9 +469,6 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
}
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
if method == "ScannerScopedDirtyUsageAck" {
return scoped_ack_matches(body, Some(scope));
}
if method == "RecoverMultipartSource" {
let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
@@ -544,9 +525,7 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
let args = match method {
"Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
"LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
_ => {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
}
_ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
};
args.into_iter()
.filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
@@ -561,39 +540,18 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
.collect()
}
fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
}
async fn forward_request(
request: Request<Incoming>,
client: ProxyClient,
target: SocketAddr,
pause: Option<Arc<PauseState>>,
ack: Option<Arc<scoped_ack::QueueState>>,
) -> io::Result<hyper::Response<ProxyBody>> {
let request = request.map(|body| body.map_err(io::Error::other).boxed());
let request = if let Some(ack) = ack
&& request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
{
match scoped_ack::intercept(request, client.clone(), target, ack).await? {
scoped_ack::Interception::Pass(request) => request,
scoped_ack::Interception::Handled(response) => return Ok(response),
}
} else {
request
};
let (parts, body) = request.into_parts();
let mut body = body;
let (mut parts, body) = request.into_parts();
let mut body = body.map_err(io::Error::other).boxed();
let mut observed = None;
let mut inspection = None;
if let Some(pause) = pause {
let method = parts
.uri
.path()
.strip_prefix("/node_service.NodeService/")
.or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
let mut route = None;
let mut lock_owners = Vec::new();
@@ -641,16 +599,6 @@ async fn forward_request(
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
.boxed();
matches
} else if method == "ScannerScopedDirtyUsageAck" {
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let matches = scoped_ack_matches(&bytes, None)?;
body = Full::new(bytes)
.map_err(|never| match never {})
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
.boxed();
matches
} else {
true
};
@@ -694,7 +642,24 @@ async fn forward_request(
}
}
}
let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
if !parts.headers.contains_key(http::header::HOST)
&& let Some(authority) = parts.uri.authority()
{
parts.headers.insert(
http::header::HOST,
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
);
}
parts.uri = http::Uri::builder()
.scheme("http")
.authority(target.to_string())
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
.build()
.map_err(io::Error::other)?;
let response = client
.request(Request::from_parts(parts, body))
.await
.map_err(io::Error::other)?;
if let Some((pause, disk)) = inspection
&& response.status().is_success()
&& response.headers().get("grpc-status").is_some_and(|status| status == "12")
@@ -747,32 +712,6 @@ async fn forward_request(
Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
}
async fn forward_to_backend(
request: Request<ProxyBody>,
client: ProxyClient,
target: SocketAddr,
) -> io::Result<hyper::Response<Incoming>> {
let (mut parts, body) = request.into_parts();
if !parts.headers.contains_key(http::header::HOST)
&& let Some(authority) = parts.uri.authority()
{
parts.headers.insert(
http::header::HOST,
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
);
}
parts.uri = http::Uri::builder()
.scheme("http")
.authority(target.to_string())
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
.build()
.map_err(io::Error::other)?;
client
.request(Request::from_parts(parts, body))
.await
.map_err(io::Error::other)
}
impl Drop for RpcFaultProxy {
fn drop(&mut self) {
if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
@@ -792,7 +731,6 @@ mod tests {
struct Backend {
received: Arc<AtomicUsize>,
auth_headers: Arc<Mutex<Vec<(String, String)>>>,
address: SocketAddr,
task: JoinHandle<()>,
}
@@ -816,8 +754,6 @@ mod tests {
let address = listener.local_addr()?;
let received = Arc::new(AtomicUsize::new(0));
let received_task = received.clone();
let auth_headers = Arc::new(Mutex::new(Vec::new()));
let backend_auth = auth_headers.clone();
let task = tokio::spawn(async move {
let mut children = JoinSet::new();
loop {
@@ -825,12 +761,10 @@ mod tests {
accepted = listener.accept() => {
let Ok((socket, _)) = accepted else { break };
let received = received_task.clone();
let backend_auth = backend_auth.clone();
let reply = reply.clone();
children.spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
let received = received.clone();
let backend_auth = backend_auth.clone();
let reply = reply.clone();
async move {
received.fetch_add(1, Ordering::SeqCst);
@@ -839,10 +773,6 @@ mod tests {
http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
})
});
if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
backend_auth.lock().expect("backend auth header observations").push((
nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
}
let body = request.into_body().collect().await?.to_bytes();
let body = reply.unwrap_or(body);
let mut trailers = http::HeaderMap::new();
@@ -865,12 +795,7 @@ mod tests {
}
}
});
Ok(Backend {
received,
auth_headers,
address,
task,
})
Ok(Backend { received, address, task })
}
async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
@@ -884,14 +809,9 @@ mod tests {
bytes: Bytes,
) -> io::Result<hyper::Response<Incoming>> {
let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
let service = if method == "ScannerScopedDirtyUsageAck" {
"ScannerControlService"
} else {
"NodeService"
};
let request = Request::builder()
.method("POST")
.uri(format!("http://{address}/node_service.{service}/{method}"))
.uri(format!("http://{address}/node_service.NodeService/{method}"))
.header(http::header::TE, "trailers")
.body(body)
.map_err(io::Error::other)?;
@@ -906,353 +826,6 @@ mod tests {
frame.into()
}
const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
fn ack_scope() -> ScopedAckScope {
ScopedAckScope {
nonce: uuid::Uuid::new_v4().to_string(),
owner_id: ACK_OWNER.to_string(),
instance_id: ACK_INSTANCE.to_string(),
buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
}
}
fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
ScannerScopedDirtyUsageAckRequest {
challenge: vec![7; 16].into(),
protocol_version: 1,
owner_id: ACK_OWNER.to_string(),
instance_id: ACK_INSTANCE.to_string(),
scope: 1,
probe_only: false,
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: bucket.to_string(),
bucket_incarnation: vec![4; 16].into(),
generation,
}],
}
}
fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
let payload = request.encode_to_vec();
let mut frame = vec![0];
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
frame.extend_from_slice(&payload);
Bytes::from(frame)
}
fn ack_hash(bytes: &[u8]) -> String {
use sha2::Digest as _;
hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
}
// Transport-only fixtures: the distributed tests supply production signatures.
async fn call_ack_frame(
client: &ProxyClient,
address: SocketAddr,
frame: Bytes,
nonce: &str,
) -> io::Result<hyper::Response<Incoming>> {
let request = Request::builder()
.method("POST")
.uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
.header(http::header::TE, "trailers")
.header("x-rustfs-rpc-nonce", nonce)
.header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
.body(Full::new(frame).map_err(|never| match never {}).boxed())
.map_err(io::Error::other)?;
client.request(request).await.map_err(io::Error::other)
}
#[tokio::test]
async fn scoped_ack_queue_loss_preserves_backend_boundary() {
tokio::time::timeout(Duration::from_secs(10), async {
for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let frame = ack_frame(&ack_request("cold-bucket", 7));
let address = proxy.local_addr();
let body = frame.clone();
let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
queue.wait_for_packets(1).await.expect("captured ACK");
assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
queue.settle(0, fault).expect("settle actual retained packet");
assert!(
caller.await.expect("caller task").is_err(),
"the caller must observe the injected transport failure"
);
let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
let response_loss = fault == ScopedAckFault::ResponseLoss;
assert_eq!(receipt["forwarded"], response_loss);
assert_eq!(receipt["backend_complete"], response_loss);
assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
if response_loss {
assert_eq!(receipt["outcome"], "response_lost");
assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
assert_eq!(receipt["response"]["bytes"], frame.len());
assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
assert!(
receipt["response"]["trailers_sha256"]
.as_str()
.is_some_and(|value| value.len() == 64)
);
} else {
assert_eq!(receipt["outcome"], "request_lost");
}
}
})
.await
.expect("bounded ACK loss checks");
}
#[tokio::test]
async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let first = ack_frame(&ack_request("cold-bucket", 7));
let second = ack_frame(&ack_request("cold-bucket", 9));
let address = proxy.local_addr();
let old_client = client.clone();
let old_body = first.clone();
let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
queue.wait_for_packets(1).await.expect("first retained packet");
old_caller.abort();
assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
let new_body = second.clone();
let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
queue.wait_for_packets(2).await.expect("second retained packet");
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
let response = new_caller
.await
.expect("new caller task")
.expect("new response")
.into_body()
.collect()
.await
.expect("complete body");
assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
assert_eq!(response.to_bytes(), second);
queue.wait_for_completion(1).await.expect("new packet complete");
queue
.settle(0, ScopedAckFault::Forward)
.expect("release old packet after its caller disappeared");
let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
assert_eq!(old["backend_complete"], true);
assert_eq!(old["request_sha256"], ack_hash(&first));
assert_eq!(old["response"]["sha256"], ack_hash(&first));
assert_eq!(backend.received.load(Ordering::SeqCst), 2);
assert_eq!(
*backend.auth_headers.lock().expect("backend header observations"),
vec![
("second".to_string(), "opaque-signature-second".to_string()),
("first".to_string(), "opaque-signature-first".to_string())
]
);
assert!(
queue.settle(0, ScopedAckFault::Forward).is_err(),
"one original packet may only be forwarded once"
);
})
.await
.expect("bounded detached retention check");
}
#[tokio::test]
async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
let mut invalid = ack_scope();
invalid.nonce.clear();
assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
let mut scope = ack_scope();
scope.buckets.insert("cold-bucket-z".to_string());
let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let mut probe = ack_request("cold-bucket", 7);
probe.probe_only = true;
let mut other_peer = ack_request("cold-bucket", 7);
other_peer.owner_id = uuid::Uuid::new_v4().to_string();
let mut other_instance = ack_request("cold-bucket", 7);
other_instance.instance_id = "b".repeat(32);
let mut invalid_challenge = ack_request("cold-bucket", 7);
invalid_challenge.challenge = vec![7; 15].into();
let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
let unselected = [
probe,
other_peer,
other_instance,
ack_request("cold-bucket-other", 7),
ack_request("cold-bucket", u64::MAX),
invalid_challenge,
noncanonical_entries,
];
let unselected_count = unselected.len();
for (index, request) in unselected.into_iter().enumerate() {
let body = ack_frame(&request);
let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
.await
.expect("unselected ACK progresses");
assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
}
assert!(queue.receipts().is_empty());
let address = proxy.local_addr();
let held_client = client.clone();
let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
let held_body = valid_frame.clone();
let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
queue.wait_for_packets(1).await.expect("one matching packet");
assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
.await
.expect("capacity overflow passes without retention");
extra.into_body().collect().await.expect("overflow response");
assert_eq!(queue.receipts().len(), 1);
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
drop(queue);
assert!(
held.await.expect("held caller").is_err(),
"dropping the session must cancel its retained packet"
);
assert_eq!(
backend.received.load(Ordering::SeqCst),
unselected_count + 1,
"session drop must never forward the held packet"
);
})
.await
.expect("bounded scope and lifecycle checks");
}
#[tokio::test]
async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let address = proxy.local_addr();
let caller = tokio::spawn(async move {
call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
});
queue.wait_for_packets(1).await.expect("retained packet");
(backend, proxy, queue, caller)
})
.await
.expect("bounded queue setup");
tokio::time::pause();
tokio::time::advance(Duration::from_secs(120)).await;
tokio::time::resume();
assert!(
tokio::time::timeout(Duration::from_secs(10), caller)
.await
.expect("expiry completes caller")
.expect("caller task")
.is_err()
);
let receipt = queue.receipts().into_iter().next().expect("expired receipt");
assert_eq!(receipt["completed"], true);
assert_eq!(receipt["forwarded"], false);
assert_eq!(receipt["backend_complete"], false);
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
assert_eq!(
queue
.settle(0, ScopedAckFault::Forward)
.expect_err("expired packet cannot be released")
.kind(),
io::ErrorKind::TimedOut
);
}
#[tokio::test]
async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
tokio::time::timeout(Duration::from_secs(10), async {
let backend = backend().await.expect("backend listener");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy listener");
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let pause = proxy
.pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
.expect("scoped ACK fault");
let frame = |bucket: &str, probe_only: bool| {
let payload = ScannerScopedDirtyUsageAckRequest {
challenge: vec![3; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
instance_id: "a".repeat(32),
scope: 1,
probe_only,
entries: vec![ScannerScopedDirtyUsageEntry {
bucket: bucket.to_string(),
bucket_incarnation: vec![4; 16].into(),
generation: 17,
}],
}
.encode_to_vec();
let mut frame = vec![0];
frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
frame.extend_from_slice(&payload);
Bytes::from(frame)
};
for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
.await
.expect("probe and other bucket must progress");
assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
}
assert_eq!(pause.hits(), 0);
let signed_bytes = frame("cold-bucket", false);
let task_bytes = signed_bytes.clone();
let address = proxy.local_addr();
let blocked =
tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
pause
.wait_for_hits(1)
.await
.expect("real ScannerControlService method must reach the fault");
assert!(!blocked.is_finished());
assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
pause.release();
let collected = blocked
.await
.expect("ACK task")
.expect("released ACK")
.into_body()
.collect()
.await
.expect("ACK body and trailers");
assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
})
.await
.expect("scanner ACK fault deadline");
}
fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
use crate::storage_api::multipart_source_rpc::{
MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
@@ -2146,4 +1719,3 @@ mod tests {
);
}
}
// event-drill: isolated harness-only change
@@ -1,404 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
use bytes::Bytes;
use http_body_util::{BodyExt as _, Full, Limited};
use hyper::{Request, Response};
use prost::Message as _;
use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
use serde_json::{Value, json};
use sha2::{Digest as _, Sha256};
use std::{
collections::BTreeSet,
io,
net::SocketAddr,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use tokio::{
sync::{Notify, oneshot, watch},
task::AbortHandle,
time::Instant,
};
const MAX_RETAINED: usize = 4;
const MAX_BYTES: usize = 1024 * 1024;
const RETENTION: Duration = Duration::from_secs(120);
pub(crate) struct ScopedAckScope {
pub nonce: String,
pub owner_id: String,
pub instance_id: String,
pub buckets: BTreeSet<String>,
}
impl ScopedAckScope {
fn validate(&self) -> io::Result<()> {
let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
if !valid_uuid(&self.nonce)
|| !valid_uuid(&self.owner_id)
|| self.instance_id.is_empty()
|| self.instance_id.len() > 128
|| self.buckets.is_empty()
|| self.buckets.len() > 16
|| self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
{
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"scoped ACK queue needs a bounded session, receiver and bucket scope",
));
}
Ok(())
}
fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
!request.probe_only
&& validate_scoped_dirty_usage_request(request).is_ok()
&& request.owner_id == self.owner_id
&& request.instance_id == self.instance_id
&& request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScopedAckFault {
Forward,
RequestLoss,
ResponseLoss,
}
struct Packet {
action: watch::Sender<Option<ScopedAckFault>>,
receipt: Mutex<Value>,
task: Mutex<Option<AbortHandle>>,
}
pub(super) struct QueueState {
scope: ScopedAckScope,
capacity: usize,
started: Instant,
deadline: Instant,
closed: AtomicBool,
// The packet inventory lock is released before any packet lock or await.
packets: Mutex<Vec<Arc<Packet>>>,
changed: Notify,
}
pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
pub(super) fn empty_slot() -> QueueSlot {
Arc::new(Mutex::new(None))
}
pub(crate) struct ScopedAckQueue {
state: Arc<QueueState>,
slot: QueueSlot,
}
impl ScopedAckQueue {
pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
scope.validate()?;
if !(1..=MAX_RETAINED).contains(&capacity) {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
}
let mut active = slot.lock().expect("scoped ACK queue slot");
if active.is_some() {
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
}
let started = Instant::now();
let state = Arc::new(QueueState {
scope,
capacity,
started,
deadline: started + RETENTION,
closed: AtomicBool::new(false),
packets: Mutex::new(Vec::new()),
changed: Notify::new(),
});
*active = Some(state.clone());
drop(active);
Ok(Self { state, slot })
}
pub(crate) fn receipts(&self) -> Vec<Value> {
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
packets
.iter()
.map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
.collect()
}
pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
self.wait_until(|| self.receipts().len() >= count).await
}
pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
self.wait_until(|| {
self.receipts()
.get(id)
.is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
})
.await?;
self.receipts()
.get(id)
.cloned()
.ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
}
async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
tokio::time::timeout_at(self.state.deadline, async {
loop {
let changed = self.state.changed.notified();
if ready() {
return;
}
changed.await;
}
})
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
}
pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
if Instant::now() >= self.state.deadline {
return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
}
let packet = self
.state
.packets
.lock()
.expect("scoped ACK packets")
.get(id)
.cloned()
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
if !packet.action.send_if_modified(|current| {
if current.is_none() {
*current = Some(fault);
true
} else {
false
}
}) {
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
}
Ok(())
}
}
impl Drop for ScopedAckQueue {
fn drop(&mut self) {
self.state.closed.store(true, Ordering::SeqCst);
let mut slot = self.slot.lock().expect("scoped ACK queue slot");
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
*slot = None;
}
drop(slot);
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
for packet in packets {
if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
task.abort();
}
}
}
}
pub(super) enum Interception {
Pass(Request<ProxyBody>),
Handled(Response<ProxyBody>),
}
fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
Full::new(bytes)
.map_err(|never| match never {})
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
.boxed()
}
fn hash(bytes: &[u8]) -> String {
hex(Sha256::digest(bytes).as_ref())
}
fn hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
}
fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
trailers.map_or(Value::Null, |headers| {
json!(
headers
.iter()
.map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
.collect::<Vec<_>>()
)
})
}
fn header_hash(headers: &http::HeaderMap) -> String {
let mut hasher = Sha256::new();
for (name, value) in headers {
hasher.update(name.as_str().as_bytes());
hasher.update([0]);
hasher.update(value.as_bytes());
hasher.update([0]);
}
hex(hasher.finalize().as_ref())
}
fn millis(state: &QueueState) -> u64 {
u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
}
pub(super) async fn intercept(
request: Request<ProxyBody>,
client: ProxyClient,
target: SocketAddr,
state: Arc<QueueState>,
) -> io::Result<Interception> {
if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
return Ok(Interception::Pass(request));
}
let (parts, body) = request.into_parts();
let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
return Ok(Interception::Pass(request));
}
let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
let (action, mut selected) = watch::channel(None);
let packet = {
let mut packets = state.packets.lock().expect("scoped ACK packets");
if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
return Ok(Interception::Pass(request));
}
let id = packets.len();
let packet = Arc::new(Packet {
action,
task: Mutex::new(None),
receipt: Mutex::new(json!({
"id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
"authority": request.uri().authority().map(|value| value.as_str()),
"captured_at_ms": millis(&state), "completed": false, "forwarded": false,
"backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
"request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
"request_trailers_sha256": trailers.as_ref().map(header_hash),
"request_trailers": trailer_receipt(trailers.as_ref()),
"auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
"signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
"replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
"request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
"probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
"bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
"generation": entry.generation })).collect::<Vec<_>>() }
})),
});
packets.push(packet.clone());
packet
};
let (reply, response) = oneshot::channel();
let worker_packet = packet.clone();
let worker_state = state.clone();
// This owner survives an HTTP/2 stream reset from the original 5 s caller.
// Every request is settled once and is discarded at the session deadline.
let task = tokio::spawn(async move {
let result = tokio::time::timeout_at(worker_state.deadline, async {
let fault = loop {
if let Some(fault) = *selected.borrow_and_update() {
break fault;
}
selected
.changed()
.await
.map_err(|_| io::Error::other("scoped ACK queue closed"))?;
};
if fault == ScopedAckFault::RequestLoss {
worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
}
if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
return Err(io::Error::new(
io::ErrorKind::ConnectionAborted,
"scoped ACK session ended or expired before forwarding",
));
}
{
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
receipt["forwarded"] = json!(true);
receipt["forwarded_at_ms"] = json!(millis(&worker_state));
}
let backend = forward_to_backend(request, client, target).await?;
let (parts, body) = backend.into_parts();
let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
let trailers = collected.trailers().cloned();
let bytes = collected.to_bytes();
let grpc_status = trailers
.as_ref()
.and_then(|values| values.get("grpc-status"))
.or_else(|| parts.headers.get("grpc-status"))
.and_then(|value| value.to_str().ok());
let response = unary_payload(&bytes)
.ok()
.and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
.map(|response| {
json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
"supported": response.supported, "cleared": response.cleared,
"response_proof_bytes": response.response_proof.len()})
});
{
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
receipt["backend_complete"] = json!(true);
receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
"bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
"trailers": trailer_receipt(trailers.as_ref()),
"trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
"response_lost"
} else {
"forwarded"
});
}
if fault == ScopedAckFault::ResponseLoss {
return Err(io::Error::new(
io::ErrorKind::ConnectionAborted,
"injected scoped ACK response loss after backend trailers",
));
}
Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
})
.await
.unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
{
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
if result.is_err() && receipt["outcome"] == "held" {
receipt["outcome"] = json!("transport_failed_or_expired");
}
receipt["completed"] = json!(true);
receipt["completed_at_ms"] = json!(millis(&worker_state));
}
let _ = reply.send(result);
worker_state.changed.notify_one();
});
*packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
if state.closed.load(Ordering::SeqCst) {
task.abort();
}
drop(task);
state.changed.notify_one();
response
.await
.map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
.map(Interception::Handled)
}
-14
View File
@@ -70,9 +70,6 @@ pub(crate) mod multipart_rpc_delay {
#[cfg(test)]
pub(crate) mod multipart_fence_acceptance {
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
/// Targeted lock diagnostics for the metadata-delay quorum case.
pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
}
#[cfg(test)]
@@ -83,14 +80,3 @@ pub(crate) mod multipart_source_rpc {
PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
};
}
#[cfg(test)]
pub(crate) mod scanner_peer_bucket {
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::erasure as ecstore_erasure;
use rustfs_ecstore::api::rpc as ecstore_rpc;
pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
pub(crate) use ecstore_rpc::PeerRestClient;
}
@@ -1112,9 +1112,8 @@ async fn wait_for_upgrade_writes(
match response {
Ok(Ok(_)) => break,
Ok(Err(error)) => {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
{
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
}
+3 -5
View File
@@ -225,11 +225,9 @@ metrics = { workspace = true }
# The guard scripts/check_no_tokio_io_uring.sh allows an explicit
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
[target.'cfg(target_os = "linux")'.dependencies]
# Native completion owners keep physical I/O leases alive after cancellation.
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
[target.'cfg(target_os = "linux")'.dev-dependencies]
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
# SharedReadBudget and the bounded read APIs are available in the 0.3.0
# registry release; keep the application on the published contract.
rustfs-uring = "0.3.0"
[target.'cfg(windows)'.dependencies]
winapi-util.workspace = true
+4 -13
View File
@@ -447,9 +447,6 @@ pub mod data_usage {
}
pub mod disk {
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
pub use crate::disk::disk_store::get_object_disk_read_timeout;
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
#[cfg(feature = "test-util")]
@@ -478,10 +475,6 @@ pub mod disk {
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
validate_batch_read_version_item_count,
};
pub use crate::disk::{
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
pub use error::DiskError;
@@ -622,7 +615,6 @@ pub mod rio {
}
pub mod rpc {
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
@@ -632,8 +624,8 @@ pub mod rpc {
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
@@ -683,9 +675,8 @@ pub mod storage {
pub use crate::store::HealWalkVersion;
pub use crate::store::{
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
};
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
}
+2 -6
View File
@@ -1503,11 +1503,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
bucket: &str,
parse: bool,
) -> Result<(BucketMetadata, bool)> {
let (metadata_res, incarnation_res) = tokio::join!(
Box::pin(read_bucket_metadata(api.clone(), bucket)),
Box::pin(load_bucket_incarnation(api.clone(), bucket)),
);
let (mut bm, persisted) = match metadata_res {
let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
Ok(res) => (res, true),
Err(err) => {
if err != Error::ConfigNotFound {
@@ -1518,7 +1514,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
}
};
let incarnation = incarnation_res?;
let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
if persisted {
if bm.name.is_empty() {
bm.name = bucket.to_string();
+21 -151
View File
@@ -1435,7 +1435,6 @@ pub(crate) async fn get_object_lock_config_state_in(
/// Re-read the Object Lock state and bucket incarnation from the same
/// authoritative metadata blob while the caller holds the transaction lock.
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
@@ -1454,27 +1453,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
get_delete_config_from_disk_in(ctx, bucket, guard, None).await
}
#[allow(clippy::type_complexity)]
pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(
ObjectLockConfigState,
Uuid,
OffsetDateTime,
std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
)> {
let (state, incarnation_id, updated_at, quota_res) =
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
Ok((
state,
incarnation_id,
updated_at,
quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
))
}
/// Validate queued expiration against the same fresh metadata used for Object
/// Lock admission. The caller holds the transaction read lock through commit,
/// so a successful lifecycle update cannot be followed by an old-policy delete.
@@ -1484,24 +1462,6 @@ pub(crate) async fn get_delete_config_from_disk_in(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
let (state, incarnation_id, updated_at, _) =
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
Ok((state, incarnation_id, updated_at))
}
#[allow(clippy::type_complexity)]
async fn get_delete_and_quota_config_from_disk_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
include_quota: bool,
) -> Result<(
ObjectLockConfigState,
Uuid,
OffsetDateTime,
Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
)> {
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
@@ -1517,7 +1477,7 @@ async fn get_delete_and_quota_config_from_disk_in(
.await?
}
};
match &authority {
match authority {
BucketMetadataAuthority::Authoritative(metadata)
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
{
@@ -1529,11 +1489,11 @@ async fn get_delete_and_quota_config_from_disk_in(
}
}
let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
let incarnation_id = metadata.bucket_incarnation_id;
let quota_res = include_quota
.then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
Ok((
object_lock_config_state_from_authoritative_metadata(&metadata)?,
metadata.bucket_incarnation_id,
object_lock_config_updated_at,
))
}
BucketMetadataAuthority::Authoritative(_) => {
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
@@ -1604,23 +1564,6 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
return Err(Error::other("quota snapshot metadata transaction fence was released"));
}
#[cfg(test)]
let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
#[cfg(not(test))]
let has_injected_error = false;
if !ctx.lock_manager().is_disabled()
&& !has_injected_error
&& let Some(fresh_quota) = snapshot.take_fresh_put_quota()
{
if snapshot_guard.is_lock_lost()
|| snapshot_guard.is_released()
|| !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
{
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
}
return fresh_quota.map_err(Error::other);
}
// The snapshot carries the caller's lifecycle fence, excluding deletion.
// Its transaction guard excludes config writes, so a same-name object
// owner need not recursively acquire the bucket namespace here.
@@ -1807,22 +1750,9 @@ struct MetadataPublishGuard {
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
}
static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
pub(crate) fn bump_local_meta_mutation_gen() {
LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
}
pub(crate) fn current_local_meta_mutation_gen() -> u64 {
LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
}
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
#[derive(Debug, Clone)]
pub struct BucketMetadataSys {
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
local_authority_cache: LocalAuthorityCache,
/// Serializes metadata-map commits and their derived cache updates for one
/// bucket. Namespace locks, when present, are acquired before this lock.
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
@@ -1866,7 +1796,6 @@ impl BucketMetadataSys {
pub fn new(api: Arc<ECStore>) -> Self {
Self {
metadata_map: Arc::new(RwLock::new(HashMap::new())),
local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
locks: StdMutex::new(HashMap::new()),
}),
@@ -1901,15 +1830,6 @@ impl BucketMetadataSys {
self.api.upgrade()
}
/// Whether the backing `ECStore` has been dropped.
///
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
/// `ECStore::drop` so a shared bootstrap context does not retain a
/// `BucketMetadataSys` whose weak handle is dangling.
pub(crate) fn is_store_stale(&self) -> bool {
self.api.upgrade().is_none()
}
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
}
@@ -2240,7 +2160,6 @@ impl BucketMetadataSys {
}
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
bump_local_meta_mutation_gen();
if !is_meta_bucketname(&bucket) {
let publish_lock = self.metadata_publish_lock(&bucket);
let _publish_guard = publish_lock.lock().await;
@@ -2259,7 +2178,6 @@ impl BucketMetadataSys {
///
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
pub async fn remove(&self, bucket: &str) -> bool {
bump_local_meta_mutation_gen();
if is_meta_bucketname(bucket) {
return false;
}
@@ -2280,7 +2198,6 @@ impl BucketMetadataSys {
}
async fn _reset(&mut self) {
bump_local_meta_mutation_gen();
let mut map = self.metadata_map.write().await;
map.clear();
drop(map);
@@ -3084,77 +3001,30 @@ impl BucketMetadataSys {
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
}
let store = self.object_store();
let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
let gen_before = current_local_meta_mutation_gen();
let cached_authority = can_use_local_authority_cache
.then(|| {
self.local_authority_cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(bucket)
.filter(|(cached_gen, _)| *cached_gen == gen_before)
.map(|(_, bm)| Arc::clone(bm))
})
.flatten();
if let Some(cached) = cached_authority {
await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata authoritative snapshot",
std::future::ready(Ok::<_, Error>(())),
)
.await?;
if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
return Ok(BucketMetadataAuthority::Authoritative(cached));
}
}
let (bucket_info_res, metadata_res) = tokio::join!(
Box::pin(await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata snapshot existence check",
async {
store
.get_bucket_info_from_sets_at_read_quorum(
bucket,
&crate::storage_api_contracts::bucket::BucketOptions::default(),
)
.await
},
)),
Box::pin(await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata authoritative snapshot",
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
)),
);
match bucket_info_res {
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
self.object_store()
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
.await
})
.await
{
Ok(_) => {}
Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
Err(err) => return Err(err),
}
let (metadata, persisted) = metadata_res?;
let (metadata, persisted) = await_bucket_namespace_operation(
Some(guard),
bucket,
"bucket metadata authoritative snapshot",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
return Err(Error::ErasureWriteQuorum);
}
if persisted {
let metadata = Arc::new(metadata);
if can_use_local_authority_cache
&& metadata.bucket_incarnation_sidecar
&& !metadata.bucket_incarnation_id.is_nil()
&& current_local_meta_mutation_gen() == gen_before
&& store.all_disks_local_and_online()
{
self.local_authority_cache
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
}
Ok(BucketMetadataAuthority::Authoritative(metadata))
Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
} else {
Ok(BucketMetadataAuthority::Fabricated)
}
@@ -3282,18 +3282,10 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
let mut join_set = JoinSet::new();
let mut all_succeeded = true;
let mut target_infos = Vec::with_capacity(tgt_arns.len());
for arn in tgt_arns {
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
all_succeeded = false;
target_infos.push(force_delete_target_info(
dobj.op_type,
&arn,
"",
false,
Err(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
));
debug!(
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -3319,7 +3311,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
let bucket = bucket.clone();
let object_name = object_name.clone();
let op_type = dobj.op_type;
join_set.spawn(async move {
if ReplicationTargetStore::target_is_offline(&tgt_client).await {
@@ -3344,16 +3335,9 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
return force_delete_target_info(
op_type,
&tgt_client.arn,
&tgt_client.endpoint,
tgt_client.secure,
Err(FORCE_DELETE_TARGET_OFFLINE_ERROR.to_string()),
);
return false;
}
let started = Instant::now();
if let Err(e) = tgt_client
.remove_object(&tgt_client.bucket, &object_name, None, replication_force_delete_remove_options())
.await
@@ -3380,25 +3364,16 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
return force_delete_target_info(
op_type,
&tgt_client.arn,
&tgt_client.endpoint,
tgt_client.secure,
Err(e.to_string()),
);
return false;
}
force_delete_target_info(op_type, &tgt_client.arn, &tgt_client.endpoint, tgt_client.secure, Ok(started.elapsed()))
true
});
}
while let Some(result) = join_set.join_next().await {
match result {
Ok(info) => {
all_succeeded &= info.replication_status == ReplicationStatusType::Completed;
target_infos.push(info);
}
Ok(success) => all_succeeded &= success,
Err(error) => {
all_succeeded = false;
error!(
@@ -3415,8 +3390,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
}
}
record_force_delete_target_stats(bucket, &target_infos).await;
if all_succeeded
&& let Some(operation_id) = dobj.delete_object.force_delete_id
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
@@ -3436,53 +3409,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
all_succeeded
}
const FORCE_DELETE_TARGET_OFFLINE_ERROR: &str = "replication target is offline";
/// One target's outcome of a replicated force-delete in the shape the
/// replication statistics consume. A force-delete carries no per-version
/// replication state, so the previous status is always `Pending`: that is
/// the transition `ReplicationStats::update` counts as a failure, whereas an
/// `Empty` previous status would drop the outcome and leave `replication
/// status` reporting zero failures while the peer still holds the objects.
fn force_delete_target_info(
op_type: ReplicationType,
arn: &str,
endpoint: &str,
secure: bool,
outcome: std::result::Result<std::time::Duration, String>,
) -> ReplicatedTargetInfo {
let mut rinfo = ReplicatedTargetInfo {
arn: arn.to_string(),
op_type,
endpoint: endpoint.to_string(),
secure,
prev_replication_status: ReplicationStatusType::Pending,
..Default::default()
};
match outcome {
Ok(duration) => {
rinfo.replication_status = ReplicationStatusType::Completed;
rinfo.duration = duration;
}
Err(error) => {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(error);
}
}
rinfo
}
async fn record_force_delete_target_stats(bucket: &str, targets: &[ReplicatedTargetInfo]) {
let Some(stats) = runtime_sources::replication_stats() else {
return;
};
for target in targets {
stats
.update(bucket, target, target.replication_status.clone(), target.prev_replication_status.clone())
.await;
}
}
fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
let mut rinfo = dobj
.delete_object
@@ -7518,51 +7444,6 @@ mod tests {
server.join().expect("test HTTP server should finish");
}
/// A force-delete the peer rejects must reach the replication failure
/// counters. A replicated force-delete used to only log the rejection, so
/// `replication status` reported zero failures while the sites diverged.
#[tokio::test]
async fn force_delete_target_outcomes_reach_replication_stats() {
let stats = super::super::replication_state::ReplicationStats::new();
let denied = force_delete_target_info(
ReplicationType::Delete,
"arn:rustfs:replication::denied:bucket",
"http://peer-a:9000",
false,
Err("remove_object failed: AccessDenied".to_string()),
);
assert_eq!(denied.replication_status, ReplicationStatusType::Failed);
assert_eq!(denied.prev_replication_status, ReplicationStatusType::Pending);
assert_eq!(denied.error.as_deref(), Some("remove_object failed: AccessDenied"));
let accepted = force_delete_target_info(
ReplicationType::Heal,
"arn:rustfs:replication::accepted:bucket",
"https://peer-b:9000",
true,
Ok(std::time::Duration::from_millis(3)),
);
assert_eq!(accepted.replication_status, ReplicationStatusType::Completed);
assert!(accepted.error.is_none());
for target in [&denied, &accepted] {
stats
.update(
"bucket",
target,
target.replication_status.clone(),
target.prev_replication_status.clone(),
)
.await;
}
let bucket_stats = stats.get("bucket").await;
let denied_stat = bucket_stats.stats.get(&denied.arn).expect("denied target: stats entry");
assert_eq!((denied_stat.failed.count, denied_stat.replicated_count), (1, 0));
let accepted_stat = bucket_stats.stats.get(&accepted.arn).expect("accepted target: stats entry");
assert_eq!((accepted_stat.failed.count, accepted_stat.replicated_count), (0, 1));
}
/// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
/// The marker is verified through the latest version instead: a
/// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
@@ -747,16 +747,6 @@ async fn list_path_raw_inner(
// Start every missing head read in the same round so one stalled
// disk cannot multiply the wait budget by the erasure-set width.
// Outcomes are still consumed below in stable disk-index order.
if !revjob_rx.is_cancelled() {
for (i, reader) in readers.iter_mut().enumerate() {
if errs[i].is_none()
&& pending_entries[i].is_none()
&& let Some(Ok(Some(entry))) = reader.try_peek_buffered()
{
pending_entries[i] = Some(entry);
}
}
}
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
if errs[i].is_some() || pending_entries[i].is_some() {
return None;
@@ -888,7 +878,7 @@ async fn list_path_raw_inner(
}
if entry.name == current.name {
// If exact match, we agree.
if current.matches_strict_borrowed(&entry) {
if current.matches_strict(&entry) {
top_entries[i] = Some(entry);
agree += 1;
} else {
@@ -996,9 +986,7 @@ async fn list_path_raw_inner(
if agree == readers.len() && !opts.preserve_replica_metadata {
for r in readers.iter_mut() {
if !r.skip_current() {
let _ = r.skip(1).await;
}
let _ = r.skip(1).await;
}
if let Some(agreed_fn) = opts.agreed.as_ref() {
@@ -1021,7 +1009,7 @@ async fn list_path_raw_inner(
}
for (i, r) in readers.iter_mut().enumerate() {
if top_entries[i].is_some() && !r.skip_current() {
if top_entries[i].is_some() {
let _ = r.skip(1).await;
}
}
@@ -1,682 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use crate::disk::background_memory::BackgroundBuffer;
use std::io::{self, IoSlice};
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll, Waker, ready};
use tokio::io::{AsyncRead, AsyncWrite, DuplexStream, ReadBuf};
pub struct BackgroundPipeReader {
inner: PipeEnd,
}
pub struct BackgroundPipeWriter {
inner: PipeEnd,
}
impl BackgroundPipeWriter {
/// Capture only the payload owner, without retaining node or physical units.
pub fn memory_budget(&self) -> Option<Arc<crate::disk::BackgroundMemoryBudget>> {
match &self.inner {
PipeEnd::Foreground(_) => None,
PipeEnd::Background(state) => state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.buffer
.memory_budget(),
}
}
}
enum PipeEnd {
Foreground(DuplexStream),
Background(Arc<Mutex<PipeState>>),
}
// The pipe lock may reserve memory. The memory budget never locks a pipe.
struct PipeState {
buffer: BackgroundBuffer,
read_offset: usize,
readable: usize,
capacity: usize,
reader_closed: bool,
writer_closed: bool,
admission_failure: Option<io::ErrorKind>,
read_waker: Option<Waker>,
write_waker: Option<Waker>,
}
/// Capture only memory ownership while the request is admitted. Payload storage
/// is reserved and initialized once, before the first nonempty background write.
pub fn background_pipe(capacity: usize) -> (BackgroundPipeReader, BackgroundPipeWriter) {
let capacity = capacity.max(1);
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
match memory {
Some(memory) => {
let state = Arc::new(Mutex::new(PipeState {
buffer: BackgroundBuffer::new(Some(memory)),
read_offset: 0,
readable: 0,
capacity,
reader_closed: false,
writer_closed: false,
admission_failure: None,
read_waker: None,
write_waker: None,
}));
(
BackgroundPipeReader {
inner: PipeEnd::Background(state.clone()),
},
BackgroundPipeWriter {
inner: PipeEnd::Background(state),
},
)
}
None => {
let (reader, writer) = tokio::io::duplex(capacity);
(
BackgroundPipeReader {
inner: PipeEnd::Foreground(reader),
},
BackgroundPipeWriter {
inner: PipeEnd::Foreground(writer),
},
)
}
}
}
fn advance_ring(offset: usize, count: usize, capacity: usize) -> usize {
let to_end = capacity - offset;
if count < to_end { offset + count } else { count - to_end }
}
fn poll_background_read(state: &Mutex<PipeState>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if output.remaining() == 0 {
return Poll::Ready(Ok(()));
}
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.readable == 0 {
if let Some(kind) = state.admission_failure.take() {
drop(state);
return Poll::Ready(Err(io::Error::new(kind, "background pipe storage admission failed")));
}
if state.writer_closed {
return Poll::Ready(Ok(()));
}
let replaced = state.read_waker.replace(cx.waker().clone());
drop(state);
drop(replaced);
return Poll::Pending;
}
let length = state.readable.min(output.remaining());
let first = length.min(state.capacity - state.read_offset);
output.put_slice(&state.buffer.as_slice()[state.read_offset..state.read_offset + first]);
output.put_slice(&state.buffer.as_slice()[..length - first]);
state.read_offset = advance_ring(state.read_offset, length, state.capacity);
state.readable -= length;
let wake = state.write_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
Poll::Ready(Ok(()))
}
fn poll_background_write(state: &Mutex<PipeState>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.reader_closed || state.writer_closed {
return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into()));
}
if input.iter().all(|part| part.is_empty()) {
return Poll::Ready(Ok(0));
}
let available = state.capacity - state.readable;
if available == 0 {
let replaced = state.write_waker.replace(cx.waker().clone());
drop(state);
drop(replaced);
return Poll::Pending;
}
let capacity = state.capacity;
if let Err(error) = state.buffer.ensure_len(capacity) {
// Legacy walk completion may suppress ordinary producer errors. A new
// pipe admission failure must also reach the reader instead of EOF.
state.admission_failure = Some(error.kind());
state.writer_closed = true;
let wake = state.read_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
return Poll::Ready(Err(error));
}
let mut offset = advance_ring(state.read_offset, state.readable, capacity);
let mut written = 0;
for part in input {
let length = part.len().min(available - written);
let first = length.min(capacity - offset);
state.buffer.as_mut_slice()[offset..offset + first].copy_from_slice(&part[..first]);
state.buffer.as_mut_slice()[..length - first].copy_from_slice(&part[first..length]);
offset = advance_ring(offset, length, capacity);
written += length;
if written == available {
break;
}
}
state.readable += written;
let wake = state.read_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
Poll::Ready(Ok(written))
}
impl AsyncRead for BackgroundPipeReader {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(reader) => Pin::new(reader).poll_read(cx, output),
PipeEnd::Background(state) => {
let coop = ready!(tokio::task::coop::poll_proceed(cx));
let result = poll_background_read(state, cx, output);
if result.is_ready() {
coop.made_progress();
}
result
}
}
}
}
impl AsyncWrite for BackgroundPipeWriter {
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write(cx, input),
PipeEnd::Background(state) => {
let coop = ready!(tokio::task::coop::poll_proceed(cx));
let result = poll_background_write(state, cx, &[IoSlice::new(input)]);
if result.is_ready() {
coop.made_progress();
}
result
}
}
}
fn poll_write_vectored(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write_vectored(cx, input),
PipeEnd::Background(state) => {
let coop = ready!(tokio::task::coop::poll_proceed(cx));
let result = poll_background_write(state, cx, input);
if result.is_ready() {
coop.made_progress();
}
result
}
}
}
fn is_write_vectored(&self) -> bool {
match &self.inner {
PipeEnd::Foreground(writer) => writer.is_write_vectored(),
PipeEnd::Background(_) => true,
}
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_flush(cx),
PipeEnd::Background(_) => Poll::Ready(Ok(())),
}
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
match &mut self.get_mut().inner {
PipeEnd::Foreground(writer) => Pin::new(writer).poll_shutdown(cx),
PipeEnd::Background(state) => {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.writer_closed = true;
let wake = state.read_waker.take();
drop(state);
if let Some(wake) = wake {
wake.wake();
}
Poll::Ready(Ok(()))
}
}
}
}
impl Drop for BackgroundPipeReader {
fn drop(&mut self) {
if let PipeEnd::Background(state) = &self.inner {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.reader_closed = true;
let wake = state.write_waker.take();
let cancelled = state.read_waker.take();
drop(state);
drop(cancelled);
if let Some(wake) = wake {
wake.wake();
}
}
}
}
impl Drop for BackgroundPipeWriter {
fn drop(&mut self) {
if let PipeEnd::Background(state) = &self.inner {
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.writer_closed = true;
let wake = state.read_waker.take();
let cancelled = state.write_waker.take();
drop(state);
drop(cancelled);
if let Some(wake) = wake {
wake.wake();
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::BackgroundMemoryBudget;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
pub(super) fn budget(capacity: usize) -> Arc<BackgroundMemoryBudget> {
let budget = BackgroundMemoryBudget::new();
budget.set_sampled_limit(u64::try_from(capacity).expect("test capacity fits u64"), Duration::from_secs(30));
budget
}
pub(super) fn local_pipe(
memory: &Arc<BackgroundMemoryBudget>,
capacity: usize,
) -> (BackgroundPipeReader, BackgroundPipeWriter) {
let state = Arc::new(Mutex::new(PipeState {
buffer: BackgroundBuffer::new(Some(memory.clone())),
read_offset: 0,
readable: 0,
capacity,
reader_closed: false,
writer_closed: false,
admission_failure: None,
read_waker: None,
write_waker: None,
}));
(
BackgroundPipeReader {
inner: PipeEnd::Background(state.clone()),
},
BackgroundPipeWriter {
inner: PipeEnd::Background(state),
},
)
}
pub(super) fn write_now(writer: &mut (impl AsyncWrite + Unpin), bytes: &[u8]) -> Poll<io::Result<usize>> {
Pin::new(writer).poll_write(&mut Context::from_waker(Waker::noop()), bytes)
}
pub(super) fn read_now(reader: &mut (impl AsyncRead + Unpin), bytes: &mut [u8]) -> Poll<io::Result<usize>> {
let mut output = ReadBuf::new(bytes);
match Pin::new(reader).poll_read(&mut Context::from_waker(Waker::noop()), &mut output) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(output.filled().len())),
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
Poll::Pending => Poll::Pending,
}
}
#[test]
fn fixed_ring_allocates_lazily_once_and_keeps_last_owner_charged() {
for capacity in [1, 7, 64 * 1024, 1024 * 1024] {
let memory = budget(capacity);
let (mut reader, mut writer) = local_pipe(&memory, capacity);
assert_eq!(memory.snapshot().used, 0);
assert!(matches!(write_now(&mut writer, &[]), Poll::Ready(Ok(0))));
assert_eq!(memory.snapshot().used, 0);
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
assert_eq!(memory.snapshot().used, capacity as u64);
memory.set_sampled_limit(0, Duration::ZERO);
let mut output = [0];
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
assert_eq!(output, *b"x");
assert!(matches!(write_now(&mut writer, b"y"), Poll::Ready(Ok(1))));
drop(writer);
assert_eq!(memory.snapshot().used, capacity as u64, "reader still owns the real ring");
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
assert_eq!(output, *b"y");
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
assert_eq!(memory.snapshot().used, capacity as u64, "draining does not destroy allocation ownership");
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
}
#[test]
fn admission_error_reaches_writer_and_reader_once_without_payload() {
for sample in [None, Some((6, Duration::from_secs(30))), Some((7, Duration::ZERO))] {
let memory = BackgroundMemoryBudget::new();
if let Some((limit, validity)) = sample {
memory.set_sampled_limit(limit, validity);
}
let (mut reader, mut writer) = local_pipe(&memory, 7);
let error = match write_now(&mut writer, b"x") {
Poll::Ready(Err(e)) => e,
_ => panic!("reject before copying first payload"),
};
assert!(
crate::disk::background_memory::is_memory_pressure(&error),
"writer retains the original budget marker"
);
assert_eq!(memory.snapshot().used, 0);
let error = match read_now(&mut reader, &mut [0]) {
Poll::Ready(Err(e)) => e,
_ => panic!("reader must see a pipe admission failure"),
};
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))), "failure is delivered once");
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
drop((reader, writer));
assert_eq!(memory.snapshot().used, 0);
}
}
#[test]
fn allocation_overflow_is_a_reader_failure_without_a_leaked_reservation() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(u64::MAX, Duration::from_secs(30));
let (mut reader, mut writer) = local_pipe(&memory, usize::MAX);
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
assert_eq!(memory.snapshot().used, 0);
assert!(matches!(read_now(&mut reader,&mut[0]),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
}
#[test]
fn scalar_and_vectored_wrap_match_a_reference_queue() {
use std::collections::VecDeque;
for capacity in [1, 3, 7, 31] {
let memory = budget(capacity);
let (mut reader, mut writer) = local_pipe(&memory, capacity);
let mut expected = VecDeque::new();
let mut next = 0u8;
for turn in 0..300 {
let input: Vec<_> = (0..(turn % 11 + 1))
.map(|_| {
let byte = next;
next = next.wrapping_add(1);
byte
})
.collect();
let available = capacity - expected.len();
let written = if turn % 2 == 0 {
write_now(&mut writer, &input)
} else {
let split = input.len() / 2;
Pin::new(&mut writer).poll_write_vectored(
&mut Context::from_waker(Waker::noop()),
&[
io::IoSlice::new(&[]),
io::IoSlice::new(&input[..split]),
io::IoSlice::new(&input[split..]),
],
)
};
if available == 0 {
assert!(written.is_pending());
} else {
let accepted = input.len().min(available);
assert!(matches!(written,Poll::Ready(Ok(n)) if n==accepted));
expected.extend(&input[..accepted]);
}
let mut output = vec![0; turn % 5 + 1];
let count = output.len().min(expected.len());
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
for byte in &output[..count] {
assert_eq!(Some(*byte), expected.pop_front());
}
assert_eq!(memory.snapshot().used, capacity as u64);
}
drop(writer);
let mut output = vec![0; capacity];
let count = expected.len();
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
assert_eq!(&output[..count], expected.into_iter().collect::<Vec<_>>().as_slice());
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn reader_drop_wakes_blocked_writer_and_writer_drop_wakes_empty_reader() {
let memory = budget(7);
let (reader, mut writer) = local_pipe(&memory, 7);
writer.write_all(b"1234567").await.expect("fill the ring");
let task = tokio::spawn(async move { writer.write_all(b"blocked").await });
tokio::task::yield_now().await;
assert!(!task.is_finished());
drop(reader);
let error = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("reader drop wakes writer")
.expect("writer task joins")
.expect_err("closed reader is BrokenPipe");
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
assert_eq!(memory.snapshot().used, 0);
let (mut reader, writer) = local_pipe(&memory, 7);
let task = tokio::spawn(async move { reader.read(&mut [0]).await });
tokio::task::yield_now().await;
assert!(!task.is_finished());
drop(writer);
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("writer drop wakes reader")
.expect("reader task joins")
.expect("clean EOF"),
0
);
}
#[tokio::test]
async fn read_wakes_a_full_writer_without_changing_byte_limit() {
let memory = budget(7);
let (mut reader, mut writer) = local_pipe(&memory, 7);
writer.write_all(b"1234567").await.expect("fill initial byte limit");
let task = tokio::spawn(async move {
writer.write_all(b"x").await.expect("resume after one byte is read");
writer
});
tokio::task::yield_now().await;
assert!(!task.is_finished());
let mut first = [0];
reader.read_exact(&mut first).await.expect("read one byte");
let writer = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("read wakes writer")
.expect("writer joins");
assert_eq!(&first, b"1");
drop(writer);
let mut all = Vec::new();
reader.read_to_end(&mut all).await.expect("drain wrapped bytes");
assert_eq!(all, b"234567x");
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn shutdown_drains_then_eof_and_cancelled_writes_accept_nothing() {
let memory = budget(7);
let (mut reader, mut writer) = local_pipe(&memory, 7);
writer
.write_all(b"1234567")
.await
.expect("fill before cancelling a blocked write");
{
let blocked = writer.write(b"lost");
tokio::pin!(blocked);
assert!(futures::poll!(&mut blocked).is_pending());
}
writer.shutdown().await.expect("shutdown does not discard buffered bytes");
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
let mut out = Vec::new();
reader.read_to_end(&mut out).await.expect("drain after shutdown");
assert_eq!(out, b"1234567");
drop(reader);
assert_eq!(memory.snapshot().used, 7, "writer still holds the allocation");
drop(writer);
assert_eq!(memory.snapshot().used, 0);
}
#[test]
fn foreground_keeps_native_duplex_semantics() {
let (reader, mut writer) = background_pipe(7);
assert!(matches!(write_now(&mut writer, b"12345678"), Poll::Ready(Ok(7))));
assert!(write_now(&mut writer, b"x").is_pending());
drop(reader);
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
let (mut reader, writer) = background_pipe(7);
drop(writer);
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))));
}
#[test]
fn cancelled_reader_leaves_the_real_ring_charged_until_writer_drop() {
let memory = budget(7);
let (reader, mut writer) = local_pipe(&memory, 7);
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
drop(reader);
assert_eq!(memory.snapshot().used, 7, "the surviving writer still owns the ring allocation");
assert!(matches!(write_now(&mut writer,b"y"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
drop(writer);
assert_eq!(memory.snapshot().used, 0);
}
#[test]
fn runtime_shutdown_releases_queued_writer_but_not_the_surviving_reader() {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build isolated runtime");
let memory = budget(7);
let (reader, mut writer) = local_pipe(&memory, 7);
runtime.block_on(async {
writer.write_all(b"x").await.expect("allocate ring before runtime shutdown");
tokio::spawn(async move {
let _writer = writer;
std::future::pending::<()>().await;
});
tokio::task::yield_now().await;
});
drop(runtime);
assert_eq!(memory.snapshot().used, 7, "reader holds the actual allocation after runtime cancellation");
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn ready_operations_keep_tokio_cooperative_yields() {
let memory = budget(7);
let (_reader, mut writer) = local_pipe(&memory, 7);
let observed = Arc::new(AtomicUsize::new(0));
let flag = observed.clone();
let task = tokio::spawn(async move {
flag.store(1, Ordering::SeqCst);
});
for _ in 0..1000 {
assert_eq!(writer.write(&[]).await.expect("empty operation remains successful"), 0);
}
assert_eq!(
observed.load(Ordering::SeqCst),
1,
"ready pipe operations must not starve other ready tasks"
);
task.await.expect("observer task joins");
}
}
#[cfg(test)]
mod waker_tests {
use super::*;
use std::sync::{
Weak,
atomic::{AtomicUsize, Ordering},
};
use std::task::Wake;
struct LockCheck {
state: Weak<Mutex<PipeState>>,
wakes: Arc<AtomicUsize>,
drops: Arc<AtomicUsize>,
}
impl LockCheck {
fn unlocked(&self) {
if let Some(state) = self.state.upgrade() {
assert!(state.try_lock().is_ok(), "waker code must run outside the pipe lock");
}
}
}
impl Wake for LockCheck {
fn wake(self: Arc<Self>) {
self.unlocked();
self.wakes.fetch_add(1, Ordering::SeqCst);
}
}
impl Drop for LockCheck {
fn drop(&mut self) {
self.unlocked();
self.drops.fetch_add(1, Ordering::SeqCst);
}
}
#[test]
fn replaced_and_woken_reader_and_writer_wakers_run_outside_the_lock() {
for waiting_reader in [true, false] {
let memory = super::tests::budget(7);
let (mut reader, mut writer) = super::tests::local_pipe(&memory, 7);
let PipeEnd::Background(state) = &reader.inner else { panic!("background fixture") };
let weak = Arc::downgrade(state);
let wakes = Arc::new(AtomicUsize::new(0));
let drops = Arc::new(AtomicUsize::new(0));
if !waiting_reader {
assert!(matches!(super::tests::write_now(&mut writer, b"1234567"), Poll::Ready(Ok(7))));
}
for _ in 0..2 {
let waker = Waker::from(Arc::new(LockCheck {
state: weak.clone(),
wakes: wakes.clone(),
drops: drops.clone(),
}));
let mut cx = Context::from_waker(&waker);
if waiting_reader {
let mut bytes = [0];
assert!(
Pin::new(&mut reader)
.poll_read(&mut cx, &mut ReadBuf::new(&mut bytes))
.is_pending()
);
} else {
assert!(Pin::new(&mut writer).poll_write(&mut cx, b"x").is_pending());
}
drop(waker);
}
assert_eq!(drops.load(Ordering::SeqCst), 1, "the replaced waker has been dropped");
if waiting_reader {
assert!(matches!(super::tests::write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
} else {
assert!(matches!(super::tests::read_now(&mut reader, &mut [0]), Poll::Ready(Ok(1))));
}
assert_eq!(wakes.load(Ordering::SeqCst), 1);
assert_eq!(drops.load(Ordering::SeqCst), 2);
}
}
}
@@ -1,586 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use crate::disk::background_memory::{BackgroundMemoryBudget, Reservation};
use bytes::{Bytes, BytesMut};
use futures::Stream;
use std::io;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
use tokio_util::io::ReaderStream;
struct ReservedReadBytes {
// Free each payload view before releasing its shared allocation receipt.
bytes: Bytes,
_reservation: Arc<Reservation>,
}
impl AsRef<[u8]> for ReservedReadBytes {
fn as_ref(&self) -> &[u8] {
&self.bytes
}
}
struct ReaderBuffer {
// These initialized bytes are spare storage, not unconsumed source data.
bytes: BytesMut,
reservation: Option<Arc<Reservation>>,
budget: Arc<BackgroundMemoryBudget>,
capacity: usize,
}
impl ReaderBuffer {
fn new(budget: Arc<BackgroundMemoryBudget>, capacity: usize) -> Self {
Self {
bytes: BytesMut::new(),
reservation: None,
budget,
capacity: capacity.max(1),
}
}
fn clear(&mut self) {
self.bytes = BytesMut::new();
self.reservation = None;
}
fn ensure_readable(&mut self) -> io::Result<()> {
if !self.bytes.is_empty() {
return Ok(());
}
if self.reservation.is_some() {
if self.bytes.try_reclaim(self.capacity) {
self.bytes.resize(self.capacity, 0);
return Ok(());
}
// A live output owns this allocation. Drop our empty view before
// its receipt, then reserve independent replacement storage.
self.clear();
}
let reservation = self.budget.reserve(self.capacity)?;
let mut bytes = Vec::new();
bytes
.try_reserve_exact(self.capacity)
.map_err(|_| io::ErrorKind::OutOfMemory)?;
bytes.resize(self.capacity, 0);
// Fresh ordinary Bytes is unique and spans the complete allocation.
// Unlike from_owner output, this conversion cannot require a copy.
let bytes = Bytes::from(bytes.into_boxed_slice())
.try_into_mut()
.map_err(|_| io::Error::other("fresh reader storage was not unique"))?;
self.bytes = bytes;
self.reservation = Some(Arc::new(reservation));
Ok(())
}
fn take_bytes(&mut self, length: usize) -> io::Result<Bytes> {
let reservation = self
.reservation
.as_ref()
.cloned()
.ok_or_else(|| io::Error::other("reader storage has no memory reservation"))?;
Ok(Bytes::from_owner(ReservedReadBytes {
bytes: self.bytes.split_to(length).freeze(),
_reservation: reservation,
}))
}
}
/// RPC response chunks retain their background memory reservation until the
/// last clone or slice is dropped. Foreground readers use the existing stream.
pub struct BackgroundReaderStream<R> {
inner: ReaderState<R>,
}
enum ReaderState<R> {
Foreground(ReaderStream<R>),
Background { reader: Option<R>, buffer: ReaderBuffer },
}
impl<R: AsyncRead> BackgroundReaderStream<R> {
/// Capture memory ownership inside the admitted scope, before moving the
/// response body out of it. This stream retains no node or device capacity.
pub fn with_capacity(reader: R, capacity: usize) -> Self {
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
let inner = match memory {
Some(memory) => ReaderState::Background {
reader: Some(reader),
buffer: ReaderBuffer::new(memory, capacity),
},
None => ReaderState::Foreground(ReaderStream::with_capacity(reader, capacity)),
};
Self { inner }
}
}
impl<R: AsyncRead + Unpin> Stream for BackgroundReaderStream<R> {
type Item = io::Result<Bytes>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
match &mut this.inner {
ReaderState::Foreground(reader) => Pin::new(reader).poll_next(cx),
ReaderState::Background { reader, buffer } => {
let Some(inner) = reader.as_mut() else {
return Poll::Ready(None);
};
if let Err(error) = buffer.ensure_readable() {
*reader = None;
return Poll::Ready(Some(Err(error)));
}
let mut output = ReadBuf::new(&mut buffer.bytes);
match std::task::ready!(Pin::new(inner).poll_read(cx, &mut output)) {
Ok(()) => {
let length = output.filled().len();
if length == 0 {
buffer.clear();
*reader = None;
Poll::Ready(None)
} else {
let bytes = buffer.take_bytes(length);
if bytes.is_err() {
buffer.clear();
*reader = None;
}
Poll::Ready(Some(bytes))
}
}
Err(error) => {
buffer.clear();
*reader = None;
Poll::Ready(Some(Err(error)))
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
use futures::StreamExt;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::io::AsyncWriteExt;
struct ObservedReader {
polls: Arc<AtomicUsize>,
fail: bool,
}
impl AsyncRead for ObservedReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
self.polls.fetch_add(1, Ordering::SeqCst);
if self.fail {
Poll::Ready(Err(io::Error::from(io::ErrorKind::PermissionDenied)))
} else {
output.put_slice(b"data");
Poll::Ready(Ok(()))
}
}
}
#[tokio::test]
async fn background_frames_keep_capacity_until_the_last_slice() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(16, Duration::from_secs(30));
let (resources, gates) = test_resource_units(2);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(io::Cursor::new(b"data"), 16)
})
.await;
assert!(
gates.iter().all(|gate| gate.snapshot().active == 0),
"the stream captures only memory ownership"
);
assert_eq!(memory.snapshot().used, 0, "construction does not allocate payload storage");
let bytes = stream.next().await.expect("first frame").expect("read frame");
assert_eq!(bytes.as_ref(), b"data");
assert_eq!(memory.snapshot().used, 16, "charge full capacity rather than visible length");
let slice = bytes.slice(1..3);
memory.set_sampled_limit(0, Duration::ZERO);
drop(bytes);
drop(stream);
assert_eq!(slice.as_ref(), b"at");
assert_eq!(memory.snapshot().used, 16);
drop(slice);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn background_denial_happens_before_reading_and_never_falls_back() {
for sample in [None, Some((7, Duration::from_secs(30))), Some((8, Duration::ZERO))] {
let memory = BackgroundMemoryBudget::new();
if let Some((limit, validity)) = sample {
memory.set_sampled_limit(limit, validity);
}
let polls = Arc::new(AtomicUsize::new(0));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(
ObservedReader {
polls: polls.clone(),
fail: false,
},
8,
)
})
.await;
let error = stream
.next()
.await
.expect("allocation failure")
.expect_err("background allocation denied");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(polls.load(Ordering::SeqCst), 0, "denial cannot consume source bytes");
assert!(stream.next().await.is_none());
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn background_pending_read_keeps_storage_through_shrink_or_cancellation() {
for cancelled in [false, true] {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(16, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let (mut sender, reader) = tokio::io::duplex(4);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(reader, 16)
})
.await;
assert!(futures::poll!(stream.next()).is_pending());
memory.set_sampled_limit(0, Duration::ZERO);
assert_eq!(memory.snapshot().used, 16);
let bytes = if cancelled {
None
} else {
sender.write_all(b"data").await.expect("release pending read");
Some(
stream
.next()
.await
.expect("pending frame")
.expect("existing allocation survives shrink"),
)
};
drop(stream);
if let Some(bytes) = bytes {
assert_eq!(bytes.as_ref(), b"data");
assert_eq!(memory.snapshot().used, 16);
drop(bytes);
}
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn background_read_error_and_eof_release_storage() {
for fail in [false, true] {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
let reader: Box<dyn AsyncRead + Unpin> = if fail {
Box::new(ObservedReader {
polls: Arc::new(AtomicUsize::new(0)),
fail: true,
})
} else {
Box::new(tokio::io::empty())
};
BackgroundReaderStream::with_capacity(reader, 8)
})
.await;
if fail {
let error = stream.next().await.expect("read failure").expect_err("preserve reader error");
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
}
assert!(stream.next().await.is_none());
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn retained_frame_blocks_new_storage_until_released() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
})
.await;
let first = stream.next().await.expect("first frame").expect("fits exact budget");
assert_eq!(first.as_ref(), b"firstone");
let error = stream
.next()
.await
.expect("next allocation")
.expect_err("held frame exhausts budget");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(memory.snapshot().used, 8);
drop(stream);
drop(first);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn consumed_frames_keep_reusable_capacity_until_eof() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
})
.await;
for expected in [b"firstone", b"followup"] {
let bytes = stream.next().await.expect("next frame").expect("released previous capacity");
assert_eq!(bytes.as_ref(), expected);
assert_eq!(memory.snapshot().used, 8);
drop(bytes);
assert_eq!(memory.snapshot().used, 8, "the reusable allocation remains owned by the stream");
}
assert!(stream.next().await.is_none());
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn foreground_stream_and_unpolled_cancellation_preserve_existing_behavior() {
let mut stream = BackgroundReaderStream::with_capacity(io::Cursor::new(b"foreground"), 4);
assert!(matches!(stream.inner, ReaderState::Foreground(_)));
let mut bytes = Vec::new();
while let Some(chunk) = stream.next().await {
bytes.extend_from_slice(&chunk.expect("foreground chunk"));
}
assert_eq!(bytes, b"foreground");
let memory = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
let stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BackgroundReaderStream::with_capacity(tokio::io::empty(), 8)
})
.await;
drop(stream);
assert_eq!(memory.snapshot().used, 0);
}
}
#[cfg(test)]
mod reuse_tests {
use super::*;
use bytes::Bytes;
use futures::StreamExt;
use std::sync::Arc;
use std::time::Duration;
use std::{
io,
pin::Pin,
task::{Context, Poll},
};
use tokio::io::{AsyncRead, ReadBuf};
static DATA: [u8; 1024] = [0x37; 1024];
struct ShortReader {
offset: usize,
length: usize,
chunk: usize,
fail_at: Option<usize>,
}
impl AsyncRead for ShortReader {
fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if self.fail_at == Some(self.offset) {
return Poll::Ready(Err(io::ErrorKind::PermissionDenied.into()));
}
let n = (self.length - self.offset).min(self.chunk).min(out.remaining());
out.put_slice(&DATA[self.offset..self.offset + n]);
self.offset += n;
Poll::Ready(Ok(()))
}
}
fn budget(limit: u64) -> Arc<crate::disk::BackgroundMemoryBudget> {
let b = crate::disk::BackgroundMemoryBudget::new();
b.set_sampled_limit(limit, Duration::from_secs(30));
b
}
async fn stream<R: AsyncRead>(
budget: &Arc<crate::disk::BackgroundMemoryBudget>,
reader: R,
capacity: usize,
) -> BackgroundReaderStream<R> {
let (resources, _) = crate::disk::test_resource_units(1);
crate::disk::with_optional_resource_unit(Some(resources.with_memory_budget(budget.clone())), async {
BackgroundReaderStream::with_capacity(reader, capacity)
})
.await
}
#[tokio::test]
async fn complete_consumed_frames_reclaim_the_same_storage_after_expiry() {
let b = budget(64);
let mut s = stream(&b, io::Cursor::new(&DATA[..]), 64).await;
let mut pointer = None;
for _ in 0..16 {
let frame = s.next().await.expect("next full frame").expect("reclaimed backing");
assert_eq!(frame.len(), 64);
assert_eq!(frame.as_ref(), &DATA[..64]);
if let Some(p) = pointer {
assert_eq!(frame.as_ptr(), p);
} else {
pointer = Some(frame.as_ptr());
}
b.set_sampled_limit(0, Duration::ZERO);
drop(frame);
assert_eq!(b.snapshot().used, 64);
}
assert!(s.next().await.is_none());
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn short_live_frames_use_disjoint_tail_then_reserve_separate_storage() {
let b = budget(32);
let mut s = stream(
&b,
ShortReader {
offset: 0,
length: 32,
chunk: 4,
fail_at: None,
},
16,
)
.await;
let mut frames: Vec<Bytes> = Vec::new();
for i in 0..4 {
let frame = s.next().await.expect("short frame").expect("remaining initialized tail");
if i != 0 {
assert_eq!(frame.as_ptr(), frames[0].as_ptr().wrapping_add(i * 4));
}
assert_eq!(frame.as_ref(), &DATA[..4]);
frames.push(frame);
assert_eq!(b.snapshot().used, 16);
}
let next = s.next().await.expect("independent frame").expect("second backing fits");
assert_ne!(next.as_ptr(), frames[0].as_ptr());
assert_eq!(b.snapshot().used, 32);
drop(frames);
assert_eq!(b.snapshot().used, 16);
drop(s);
assert_eq!(b.snapshot().used, 16);
drop(next);
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn final_slice_prevents_reclaim_and_keeps_old_plus_new_charged() {
let b = budget(32);
let mut s = stream(&b, io::Cursor::new(&DATA[..32]), 16).await;
let first = s.next().await.expect("old frame").expect("old backing");
let last = first.slice(15..);
drop(first);
let next = s.next().await.expect("new frame").expect("second backing fits");
assert_eq!(last.as_ref(), &DATA[..1]);
assert_ne!(next.as_ptr(), last.as_ptr());
assert_eq!(b.snapshot().used, 32);
drop(s);
assert_eq!(b.snapshot().used, 32);
drop(next);
assert_eq!(b.snapshot().used, 16);
drop(last);
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn expiry_uses_tail_but_rejects_new_storage_before_consuming_source() {
let b = budget(16);
let mut s = stream(
&b,
ShortReader {
offset: 0,
length: 32,
chunk: 4,
fail_at: None,
},
16,
)
.await;
let first = s.next().await.expect("first frame").expect("initial backing");
b.set_sampled_limit(0, Duration::ZERO);
for _ in 0..3 {
drop(s.next().await.expect("tail frame").expect("reuse charged storage"));
}
let error = s
.next()
.await
.expect("new backing denied")
.expect_err("live frame prevents reclaim");
assert!(crate::disk::background_memory::is_memory_pressure(&error));
assert!(s.next().await.is_none());
assert_eq!(b.snapshot().used, 16);
drop(first);
assert_eq!(b.snapshot().used, 0);
}
#[tokio::test]
async fn eof_and_error_release_spare_but_keep_live_output_charged() {
for fail_at in [None, Some(4)] {
let b = budget(16);
let mut s = stream(
&b,
ShortReader {
offset: 0,
length: 4,
chunk: 4,
fail_at,
},
16,
)
.await;
let frame = s.next().await.expect("first short frame").expect("payload");
match fail_at {
None => assert!(s.next().await.is_none()),
Some(_) => assert_eq!(
s.next()
.await
.expect("read error")
.expect_err("preserve source failure")
.kind(),
io::ErrorKind::PermissionDenied
),
}
assert!(s.next().await.is_none());
assert_eq!(b.snapshot().used, 16);
assert_eq!(frame.as_ref(), &DATA[..4]);
drop(frame);
assert_eq!(b.snapshot().used, 0);
}
}
#[test]
fn runtime_shutdown_drops_a_pending_stream_without_refunding_early() {
for abort in [false, true] {
let b = budget(16);
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
let (_tx, reader) = tokio::io::duplex(4);
let (began, wait) = tokio::sync::oneshot::channel();
let memory = b.clone();
let handle = rt.spawn(async move {
let mut s = stream(&memory, reader, 16).await;
assert!(futures::poll!(s.next()).is_pending());
assert_eq!(memory.snapshot().used, 16);
began.send(()).expect("notify pending allocation");
let _ = s.next().await;
});
rt.block_on(wait).expect("stream pending");
assert_eq!(b.snapshot().used, 16);
if abort {
handle.abort();
let _ = rt.block_on(handle);
}
drop(rt);
assert_eq!(b.snapshot().used, 0);
}
}
}
+4 -13
View File
@@ -17,10 +17,10 @@ use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
use crate::cluster::rpc::http_auth::{
AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
RollingMutationBodyDigest, TIMESTAMP_HEADER, gen_tonic_replay_scope_headers_with_admission, internode_rpc_body_digest_strict,
rpc_background_admission, verify_tonic_peer_replay_capabilities_response,
RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
verify_tonic_peer_replay_capabilities_response,
};
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
#[cfg(test)]
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
@@ -415,16 +415,7 @@ where
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok()),
) {
match rpc_background_admission(request.headers()).and_then(|background| {
gen_tonic_replay_scope_headers_with_admission(
audience,
request.uri().path(),
timestamp,
content_sha256,
boot_epoch,
background,
)
}) {
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
Ok(headers) => request.headers_mut().extend(headers),
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
}
+24 -195
View File
@@ -61,9 +61,6 @@ use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
@@ -80,46 +77,6 @@ pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
tokio::task_local! {
static RPC_BACKGROUND_REQUEST: bool;
}
pub(crate) fn current_rpc_background_request() -> bool {
RPC_BACKGROUND_REQUEST
.try_with(|background| *background)
.unwrap_or_else(|_| crate::disk::has_resource_io_scope())
}
/// Preserve the authenticated wire class while polling locally admitted work.
/// This scope carries only the class, never node, disk, or memory ownership.
pub async fn with_rpc_background_request<F: std::future::Future>(background: bool, operation: F) -> F::Output {
RPC_BACKGROUND_REQUEST.scope(background, operation).await
}
/// Parse the request class only; callers must verify the signature before using it.
pub fn rpc_background_admission(headers: &HeaderMap) -> std::io::Result<bool> {
let mut values = headers.get_all(RPC_BACKGROUND_ADMISSION_HEADER).iter();
match (values.next(), values.next()) {
(None, None) => Ok(false),
(Some(value), None) if value == RPC_BACKGROUND_ADMISSION_V1 => Ok(true),
_ => Err(std::io::Error::other("Invalid RPC background admission version")),
}
}
fn bind_background_admission(mac: &mut HmacSha256, background: bool) {
if background {
// Every auth lane binds the class, so stripping the marker or falling
// back to an older signature cannot turn admitted work into foreground I/O.
mac.update(RPC_BACKGROUND_ADMISSION_DOMAIN);
}
}
fn insert_background_admission(headers: &mut HeaderMap, background: bool) {
if background {
headers.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static(RPC_BACKGROUND_ADMISSION_V1));
}
}
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
@@ -562,23 +519,21 @@ fn redacted_rpc_path(url: &str) -> String {
}
/// Generate HMAC-SHA256 signature for the given data
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64, background: bool) -> String {
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
let data = signature_payload(url, method, timestamp);
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
bind_background_admission(&mut mac, background);
mac.update(data.as_bytes());
let result = mac.finalize();
base64_simd::STANDARD.encode_to_string(result.into_bytes())
}
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str, background: bool) -> bool {
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
return false;
};
let data = signature_payload(url, method, timestamp);
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
bind_background_admission(&mut mac, background);
mac.update(data.as_bytes());
mac.verify_slice(&signature).is_ok()
}
@@ -816,7 +771,6 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
#[derive(Clone, Copy)]
struct SignatureV2Scope<'a> {
background: bool,
audience: &'a str,
service: &'a str,
rpc_method: &'a str,
@@ -826,7 +780,6 @@ struct SignatureV2Scope<'a> {
}
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
bind_background_admission(mac, scope.background);
for part in [
b"rustfs-rpc-auth-v2|".as_slice(),
scope.audience.as_bytes(),
@@ -865,7 +818,6 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
#[derive(Clone, Copy)]
struct ReplayScope<'a> {
background: bool,
audience: &'a str,
path: &'a str,
timestamp: &'a str,
@@ -875,7 +827,6 @@ struct ReplayScope<'a> {
}
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
bind_background_admission(mac, scope.background);
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
for part in [
scope.audience.as_bytes(),
@@ -1019,24 +970,6 @@ pub fn gen_tonic_replay_scope_headers(
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
) -> std::io::Result<HeaderMap> {
gen_tonic_replay_scope_headers_with_admission(
audience,
path,
timestamp,
content_sha256,
boot_epoch,
current_rpc_background_request(),
)
}
pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
audience: &str,
path: &str,
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
background: bool,
) -> std::io::Result<HeaderMap> {
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
@@ -1050,7 +983,6 @@ pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
let signature = generate_replay_scope_signature(
&get_shared_secret()?,
ReplayScope {
background,
audience,
path,
timestamp,
@@ -1060,7 +992,6 @@ pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
},
)?;
let mut headers = HeaderMap::new();
insert_background_admission(&mut headers, background);
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
headers.insert(
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
@@ -1295,12 +1226,10 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
let secret = get_shared_secret()?;
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
let background = current_rpc_background_request();
let signature = generate_signature(&secret, url, method, timestamp, background);
let signature = generate_signature(&secret, url, method, timestamp);
let mut headers = HeaderMap::new();
insert_background_admission(&mut headers, background);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).expect("Invalid header value"));
headers.insert(
TIMESTAMP_HEADER,
@@ -1335,12 +1264,10 @@ pub fn gen_tonic_signature_headers(
let timestamp_header = timestamp.to_string();
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
let background = current_rpc_background_request();
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp, background);
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
let signature_v2 = generate_signature_v2(
&secret,
SignatureV2Scope {
background,
audience,
service,
rpc_method,
@@ -1351,7 +1278,6 @@ pub fn gen_tonic_signature_headers(
)?;
let mut headers = HeaderMap::new();
insert_background_admission(&mut headers, background);
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
headers.insert(TIMESTAMP_HEADER, header_value(&timestamp_header, TIMESTAMP_HEADER)?);
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
@@ -1575,7 +1501,6 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
if !verify_replay_scope_signature(
&secret,
ReplayScope {
background: rpc_background_admission(headers)?,
audience,
path,
timestamp,
@@ -1700,9 +1625,6 @@ fn verify_tonic_rpc_signature_with_strictness(
strict: bool,
) -> std::io::Result<()> {
if !has_v2_auth_headers(headers) {
if rpc_background_admission(headers)? {
return Err(std::io::Error::other("RPC background admission requires v2 authentication"));
}
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
if strict {
@@ -1774,7 +1696,6 @@ fn verify_tonic_rpc_signature_with_strictness(
if !verify_signature_v2(
&secret,
SignatureV2Scope {
background: rpc_background_admission(headers)?,
audience,
service,
rpc_method,
@@ -1815,7 +1736,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
// Verify signature with constant-time HMAC comparison.
let secret = get_shared_secret()?;
if !verify_signature(&secret, url, method, timestamp, signature, rpc_background_admission(headers)?) {
if !verify_signature(&secret, url, method, timestamp, signature) {
let rpc_path = redacted_rpc_path(url);
error!(
rpc_path = %rpc_path,
@@ -1907,98 +1828,6 @@ mod tests {
runtime_sources::ensure_test_rpc_secret();
}
#[tokio::test]
async fn background_admission_is_bound_in_http_and_both_tonic_signature_lanes() {
ensure_test_rpc_secret();
let secret = get_shared_secret().expect("RPC secret");
let path = "/node_service.NodeService/ReadAll";
for background in [false, true] {
let (http, tonic) = with_rpc_background_request(background, async {
(
gen_signature_headers("/rustfs/rpc/read?disk=a", &Method::GET).expect("HTTP signature"),
gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "ReadAll", None)
.expect("gRPC signature"),
)
})
.await;
assert_eq!(rpc_background_admission(&http).expect("HTTP class"), background);
assert_eq!(rpc_background_admission(&tonic).expect("gRPC class"), background);
verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &http).expect("HTTP authentication");
verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &tonic, true).expect("v2 authentication");
let timestamp = tonic[TIMESTAMP_HEADER].to_str().expect("timestamp");
let content = tonic[RPC_CONTENT_SHA256_HEADER].to_str().expect("content digest");
assert_eq!(
verify_signature_v2(
&secret,
SignatureV2Scope {
background: false,
audience: "node-a:9000",
service: "node_service.NodeService",
rpc_method: "ReadAll",
timestamp,
nonce: tonic[RPC_NONCE_HEADER].to_str().expect("nonce"),
content_sha256: content,
},
tonic[RPC_SIGNATURE_V2_HEADER].to_str().expect("signature"),
),
!background,
"an old peer cannot authenticate a background request as foreground"
);
let mut replay = tonic.clone();
// The channel can run outside the caller's task-local scope.
replay.extend(
gen_tonic_replay_scope_headers_with_admission(
"node-a:9000",
path,
timestamp,
content,
tonic_rpc_boot_epoch(),
background,
)
.expect("v3 signature"),
);
for mutation in ["strip-or-add", "unknown", "duplicate"] {
let alter = |headers: &HeaderMap| {
let mut changed = headers.clone();
match mutation {
"strip-or-add" if background => {
changed.remove(RPC_BACKGROUND_ADMISSION_HEADER);
}
"strip-or-add" => insert_background_admission(&mut changed, true),
"unknown" => {
changed.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v2"));
}
_ => {
insert_background_admission(&mut changed, true);
changed.append(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v1"));
}
}
changed
};
assert!(verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &alter(&http)).is_err());
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &alter(&tonic), true).is_err());
assert!(verify_tonic_replay_scope_signature("node-a:9000", path, &alter(&replay)).is_err());
}
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).expect("v3 authentication");
assert!(
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).is_err(),
"replay rejected"
);
if background {
let mut downgraded = HeaderMap::new();
for name in [SIGNATURE_HEADER, TIMESTAMP_HEADER, RPC_BACKGROUND_ADMISSION_HEADER] {
downgraded.insert(name, tonic[name].clone());
}
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
downgraded.remove(RPC_BACKGROUND_ADMISSION_HEADER);
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
}
}
}
#[test]
fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
let secret = "test-scanner-capability-secret";
@@ -2136,8 +1965,8 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200; // Fixed timestamp
let signature1 = generate_signature(secret, url, &method, timestamp, false);
let signature2 = generate_signature(secret, url, &method, timestamp, false);
let signature1 = generate_signature(secret, url, &method, timestamp);
let signature2 = generate_signature(secret, url, &method, timestamp);
assert_eq!(signature1, signature2, "Same inputs should produce same signature");
assert!(!signature1.is_empty(), "Signature should not be empty");
@@ -2150,10 +1979,10 @@ mod tests {
let method = Method::GET;
let timestamp = 1640995200;
let signature1 = generate_signature(secret, url, &method, timestamp, false);
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp, false);
let signature3 = generate_signature(secret, url, &Method::POST, timestamp, false);
let signature4 = generate_signature(secret, url, &method, timestamp + 1, false);
let signature1 = generate_signature(secret, url, &method, timestamp);
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp);
let signature3 = generate_signature(secret, url, &Method::POST, timestamp);
let signature4 = generate_signature(secret, url, &method, timestamp + 1);
assert_ne!(signature1, signature2, "Different URLs should produce different signatures");
assert_ne!(signature1, signature3, "Different methods should produce different signatures");
@@ -2259,14 +2088,14 @@ mod tests {
let url = "http://example.com/api/test";
let method = Method::GET;
let timestamp = 1640995200;
let signature = generate_signature(secret, url, &method, timestamp, false);
let signature = generate_signature(secret, url, &method, timestamp);
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
tampered[0] ^= 1;
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
assert!(verify_signature(secret, url, &method, timestamp, &signature, false));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature, false));
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature", false));
assert!(verify_signature(secret, url, &method, timestamp, &signature));
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature"));
}
#[test]
@@ -2283,11 +2112,11 @@ mod tests {
);
let method = Method::GET;
let timestamp = 1_640_995_200;
let signature = generate_signature(secret, signed_url, &method, timestamp, false);
let signature = generate_signature(secret, signed_url, &method, timestamp);
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature, false));
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature, false));
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature, false));
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature));
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature));
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature));
}
#[test]
@@ -2297,7 +2126,7 @@ mod tests {
let method = Method::GET;
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
let secret = get_shared_secret().expect("test RPC secret should resolve");
let expected_signature = generate_signature(&secret, url, &method, timestamp, false);
let expected_signature = generate_signature(&secret, url, &method, timestamp);
let invalid_signature = "invalid-signature";
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
@@ -2335,7 +2164,7 @@ mod tests {
// Set expired timestamp (older than SIGNATURE_VALID_DURATION)
let expired_timestamp = OffsetDateTime::now_utc().unix_timestamp() - SIGNATURE_VALID_DURATION - 10;
let secret = get_shared_secret().expect("test RPC secret should resolve");
let signature = generate_signature(&secret, url, &method, expired_timestamp, false);
let signature = generate_signature(&secret, url, &method, expired_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&expired_timestamp.to_string()).unwrap());
@@ -2357,7 +2186,7 @@ mod tests {
let future_timestamp = OffsetDateTime::now_utc().unix_timestamp() + SIGNATURE_VALID_DURATION + 10;
let secret = get_shared_secret().expect("test RPC secret should resolve");
let signature = generate_signature(&secret, url, &method, future_timestamp, false);
let signature = generate_signature(&secret, url, &method, future_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&future_timestamp.to_string()).unwrap());
@@ -2471,7 +2300,7 @@ mod tests {
// Test timestamp just within valid duration
let valid_timestamp = current_time - SIGNATURE_VALID_DURATION + 1;
let signature = generate_signature(&secret, url, &method, valid_timestamp, false);
let signature = generate_signature(&secret, url, &method, valid_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&valid_timestamp.to_string()).unwrap());
@@ -2482,7 +2311,7 @@ mod tests {
// Test timestamp just outside valid duration
let mut headers = HeaderMap::new();
let invalid_timestamp = current_time - SIGNATURE_VALID_DURATION - 15;
let signature = generate_signature(&secret, url, &method, invalid_timestamp, false);
let signature = generate_signature(&secret, url, &method, invalid_timestamp);
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&invalid_timestamp.to_string()).unwrap());
@@ -33,9 +33,7 @@ use rustfs_config::{
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
};
use rustfs_rio::{
ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
};
use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::future::Future;
@@ -68,19 +66,6 @@ const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const CONTENT_TYPE_JSON: &str = "application/json";
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
fn http_writer_memory_reserver() -> Option<HttpWriterMemoryReserver> {
crate::disk::current_resource_unit()
.and_then(|resources| resources.memory_budget())
.map(|budget| {
// Payloads can outlive the operation; never retain its node or disk units.
Arc::new(move |capacity| {
budget
.reserve(capacity)
.map(|reservation| Box::new(reservation) as HttpWriterMemoryReservation)
}) as HttpWriterMemoryReserver
})
}
fn unsupported_transport_message(transport: &str) -> String {
format!(
"invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
@@ -361,17 +346,13 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
let memory_reserver = http_writer_memory_reserver();
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
let url = build_put_file_stream_url(&request, auth_scope);
let endpoint = request.endpoint;
let mut headers = json_headers();
build_auth_headers(&url, &Method::PUT, &mut headers)?;
let writer = match memory_reserver {
Some(reserve) => HttpWriter::new_with_memory_reserver(url.clone(), Method::PUT, headers, reserve).await?,
None => HttpWriter::new(url.clone(), Method::PUT, headers).await?,
};
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
match auth_scope {
Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
None => Ok(Box::new(writer)),
@@ -967,193 +948,6 @@ mod tests {
use tokio::io::AsyncWriteExt;
use tokio::sync::{Barrier, Notify};
fn memory_test_write_request(authenticated: bool) -> WriteStreamRequest {
crate::runtime::sources::ensure_test_rpc_secret();
let endpoint = format!("http://writer-memory-{}.invalid", Uuid::new_v4());
put_file_capability_cache_entry(&endpoint).write().cached = Some(if authenticated {
PutFileCapabilityState::V1 {
server_epoch: Uuid::new_v4(),
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
}
} else {
PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL)
});
WriteStreamRequest {
endpoint,
disk: "disk-a".to_owned(),
volume: "bucket".to_owned(),
path: "part.1".to_owned(),
append: false,
size: 4,
}
}
#[tokio::test]
async fn http_write_memory_reserver_outlives_only_the_memory_scope() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
assert!(http_writer_memory_reserver().is_none(), "foreground must use the existing writer");
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(2 * 1024 * 1024, Duration::from_secs(30));
let (resources, gates) = test_resource_units(2);
let request = memory_test_write_request(true);
let endpoint = request.endpoint.clone();
let writer = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
TcpHttpInternodeDataTransport.open_write(request),
)
.await;
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
let mut writer = writer.expect("admitted transport writer");
assert!(
gates.iter().all(|gate| gate.snapshot().active == 0),
"the callback must not retain disk units"
);
assert_eq!(memory.snapshot().used, 1024 * 1024);
writer.write_all(b"data").await.expect("buffered bytes");
writer.flush().await.expect("transfer charged allocation to queue");
memory.set_sampled_limit(2 * 1024 * 1024, Duration::ZERO);
writer
.write_all(&vec![0x5a; 1024 * 1024 - 4])
.await
.expect("reuse the already charged tail after observation expiry");
assert_eq!(memory.snapshot().used, 1024 * 1024, "reuse must not allocate separate storage");
let error = writer
.write(b"rejected")
.await
.expect_err("expired observation must reject new storage");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert!(
crate::disk::background_memory::is_memory_pressure(&error),
"preserve the typed memory-pressure marker"
);
assert_eq!(memory.snapshot().used, 1024 * 1024, "queued storage remains charged after expiry");
drop(writer);
tokio::task::yield_now().await;
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn http_write_memory_denial_preserves_the_budget_error() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
for (authenticated, sample) in [false, true].into_iter().flat_map(|authenticated| {
[
None,
Some((1024 * 1024 - 1, Duration::from_secs(30))),
Some((1024 * 1024, Duration::ZERO)),
]
.into_iter()
.map(move |sample| (authenticated, sample))
}) {
let memory = BackgroundMemoryBudget::new();
if let Some((limit, valid_for)) = sample {
memory.set_sampled_limit(limit, valid_for);
}
let (resources, _) = test_resource_units(1);
let request = memory_test_write_request(authenticated);
let endpoint = request.endpoint.clone();
let result = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
TcpHttpInternodeDataTransport.open_write(request),
)
.await;
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
let error = match result {
Err(Error::Io(error)) => error,
Err(error) => panic!("unexpected transport error: {error}"),
Ok(_) => panic!("unavailable memory must reject construction"),
};
assert!(crate::disk::background_memory::is_memory_pressure(&error));
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn admitted_remote_reads_bind_receiver_admission_to_the_http_signature() {
use crate::disk::with_background_resource_unit;
use rustfs_concurrency::adaptive::AdaptiveAdmission;
use std::num::NonZeroU16;
crate::runtime::sources::ensure_test_rpc_secret();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("remote admission listener");
let address = listener.local_addr().expect("remote admission address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("remote read connection");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("remote read headers");
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete request headers");
bytes.extend_from_slice(&chunk[..read]);
}
let request = std::str::from_utf8(&bytes).expect("HTTP request");
let mut lines = request.split("\r\n");
let target = lines
.next()
.expect("request line")
.split_whitespace()
.nth(1)
.expect("request target")
.to_owned();
let mut headers = HeaderMap::new();
for line in lines.take_while(|line| !line.is_empty()) {
let (name, value) = line.split_once(':').expect("header field");
headers.append(
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
HeaderValue::from_str(value.trim()).expect("header value"),
);
}
socket
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 5\r\nconnection: close\r\n\r\nhello")
.await
.expect("remote read response");
(target, headers)
});
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let resource = gate.try_acquire().expect("sender device admission");
let mut reader = tokio::time::timeout(
Duration::from_secs(5),
with_background_resource_unit(
resource,
TcpHttpInternodeDataTransport.open_read(ReadStreamRequest {
endpoint: format!("http://{address}"),
disk: "disk-a".to_owned(),
volume: "bucket".to_owned(),
path: "part.1".to_owned(),
offset: 0,
length: 5,
stall_timeout: Some(Duration::from_secs(5)),
}),
),
)
.await
.expect("bounded remote request")
.expect("remote reader");
let mut body = Vec::new();
reader.read_to_end(&mut body).await.expect("complete remote body");
assert_eq!(body, b"hello");
let (target, headers) = server.await.expect("remote request capture");
assert_eq!(gate.snapshot().active, 0, "remote body does not retain the sender's device");
assert_eq!(
headers
.get("x-rustfs-rpc-background-admission")
.and_then(|value| value.to_str().ok()),
Some("v1"),
"resource-controlled remote work must request receiver-side admission"
);
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &headers).expect("bound request signature");
let mut stripped = headers;
stripped.remove("x-rustfs-rpc-background-admission");
assert!(
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &stripped).is_err(),
"removing background admission must invalidate the signature"
);
}
async fn wait_for_capability_flight_waiters(entry: &PutFileCapabilityCacheEntry, waiters: usize) {
tokio::time::timeout(Duration::from_secs(5), async {
loop {
+10 -13
View File
@@ -13,8 +13,6 @@
// limitations under the License.
pub(crate) mod background_monitor;
pub(crate) mod background_pipe;
pub(crate) mod background_reader_stream;
pub(crate) mod client;
pub(crate) mod context_propagation;
pub(crate) mod http_auth;
@@ -35,17 +33,16 @@ pub use client::{
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
#[allow(unused_imports)]
pub use http_auth::{
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
+6 -105
View File
@@ -17,7 +17,6 @@ use crate::cluster::rpc::client::{
node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class,
node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::http_auth::{current_rpc_background_request, with_rpc_background_request};
use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
WriteStreamRequest,
@@ -571,7 +570,6 @@ struct RetryingRemoteReader {
reader: Option<FileReader>,
transport: Arc<dyn InternodeDataTransport>,
request: ReadStreamRequest,
background: bool,
emitted: usize,
retried: bool,
initial_read_timeout: Option<Duration>,
@@ -592,7 +590,6 @@ impl RetryingRemoteReader {
reader: Some(reader),
transport,
request,
background: current_rpc_background_request(),
emitted: 0,
retried: false,
initial_read_timeout,
@@ -610,10 +607,9 @@ impl RetryingRemoteReader {
let request = resumed_read_request(&self.request, self.emitted)?;
let recovery_timeout = self.recovery_timeout;
let transport = Arc::clone(&self.transport);
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
self.background,
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await },
))));
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await
})));
Ok(())
}
@@ -721,7 +717,6 @@ struct RetryingRemoteChunkReader {
reader: Option<rustfs_rio::ChunkReaderBox>,
transport: Arc<dyn InternodeDataTransport>,
request: ReadStreamRequest,
background: bool,
emitted: usize,
retried: bool,
initial_read_timeout: Option<Duration>,
@@ -742,7 +737,6 @@ impl RetryingRemoteChunkReader {
reader: Some(reader),
transport,
request,
background: current_rpc_background_request(),
emitted: 0,
retried: false,
initial_read_timeout,
@@ -760,10 +754,9 @@ impl RetryingRemoteChunkReader {
let request = resumed_read_request(&self.request, self.emitted)?;
let recovery_timeout = self.recovery_timeout;
let transport = Arc::clone(&self.transport);
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
self.background,
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await },
))));
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await
})));
Ok(())
}
@@ -7776,98 +7769,6 @@ mod tests {
.await;
}
#[tokio::test]
async fn remote_read_retries_preserve_signed_admission_after_sender_scope_ends() {
use crate::cluster::rpc::{rpc_background_admission, verify_rpc_signature};
use crate::disk::with_background_resource_unit;
use rustfs_concurrency::adaptive::AdaptiveAdmission;
use std::num::NonZeroU16;
crate::runtime::sources::ensure_test_rpc_secret();
for background in [false, true] {
for chunked in [false, true] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("retry listener");
let address = listener.local_addr().expect("retry address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("fresh connection");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("retry request");
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete headers");
bytes.extend_from_slice(&chunk[..read]);
}
let request = std::str::from_utf8(&bytes).expect("HTTP request");
let mut lines = request.split("\r\n");
let target = lines
.next()
.expect("request line")
.split_whitespace()
.nth(1)
.expect("target")
.to_owned();
let mut headers = http::HeaderMap::new();
for line in lines.take_while(|line| !line.is_empty()) {
let (name, value) = line.split_once(':').expect("header");
headers.append(
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
http::HeaderValue::from_str(value.trim()).expect("header value"),
);
}
socket
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 6\r\nconnection: close\r\n\r\n456789")
.await
.expect("retry response");
(target, headers)
});
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let request = ReadStreamRequest {
endpoint: format!("http://{address}"),
..resume_request(10)
};
let create = async {
let transport = Arc::new(TcpHttpInternodeDataTransport);
if chunked {
Box::new(RetryingRemoteChunkReader::new_with_timeouts(
resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
transport,
request,
None,
None,
)) as FileReader
} else {
Box::new(RetryingRemoteReader::new_with_timeouts(
resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
transport,
request,
None,
None,
)) as FileReader
}
};
let mut reader = if background {
with_background_resource_unit(gate.try_acquire().expect("sender admission"), create).await
} else {
create.await
};
assert_eq!(gate.snapshot().active, 0, "retry reader cannot retain sender disk capacity");
let mut output = Vec::new();
tokio::time::timeout(
Duration::from_secs(5),
with_rpc_background_request(!background, reader.read_to_end(&mut output)),
)
.await
.expect("bounded retry")
.expect("resumed body");
assert_eq!(output, b"0123456789", "chunked={chunked}, background={background}");
let (target, headers) = server.await.expect("captured retry");
assert!(target.contains("offset=11") && target.contains("length=6"), "{target}");
assert_eq!(rpc_background_admission(&headers).expect("admission marker"), background);
verify_rpc_signature(&target, &http::Method::GET, &headers).expect("signed retry class");
}
}
}
#[tokio::test]
async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));
@@ -1315,6 +1315,34 @@ impl LockClient for RemoteClient {
Ok(resp.success)
}
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
let mut request = Self::create_unlock_request(lock_id);
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
let mut client = self.get_client().await?;
let resource_summary = request.resource.to_string();
let mut req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
});
attach_lock_mutation_body_digest(&mut req)?;
let response = self
.execute_rpc(
"refresh",
&resource_summary,
Self::rpc_timeout(),
async move { client.refresh(req).await },
RpcCleanup::new(None, None, None),
)
.await?
.into_inner();
if let Some(error) = response.error_info {
return Err(LockError::internal(error));
}
// Older nodes refresh normally but return no conversion marker.
Ok(response.success && response.lock_info.as_deref() == Some(marker))
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
-1
View File
@@ -3506,7 +3506,6 @@ mod tests {
replication_decision: String::new(),
checksum: None,
multipart_completion_replayed: false,
backfilled_old_current_size: None,
}
}
}
@@ -391,7 +391,7 @@ pub(super) async fn fixture_inline_pool_meta_replicas(payload: &[u8], modified:
writer.shutdown().await.expect("fixture bitrot writer should finish");
let mut info = base.clone();
info.erasure.index = *coding_index;
info.data = Some(writer.into_inline_data().expect("fixture writer should retain inline bytes"));
info.data = Some(Bytes::from(writer.into_inline_data().expect("fixture writer should retain inline bytes")));
let mut meta = FileMeta::new();
meta.add_version(info).expect("test inline FileInfo should serialize");
raw.push(Bytes::from(meta.marshal_msg().expect("test xl.meta should serialize")));
+1 -11
View File
@@ -84,7 +84,7 @@ pub(crate) enum CrashPoint {
}
#[cfg(test)]
pub(crate) use armed::{arm, disarm, is_armed, should_crash_at};
pub(crate) use armed::{arm, disarm, should_crash_at};
/// Production no-op: the compiler inlines this to `false`, so armed crash points
/// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
@@ -94,12 +94,6 @@ pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
false
}
#[cfg(not(test))]
#[inline(always)]
pub(crate) fn is_armed(_point: CrashPoint, _key: &str) -> bool {
false
}
#[cfg(test)]
mod armed {
use super::CrashPoint;
@@ -138,10 +132,6 @@ mod armed {
guard().retain(|(p, k)| !(*p == point && k == key));
}
pub(crate) fn is_armed(point: CrashPoint, key: &str) -> bool {
guard().iter().any(|(p, k)| *p == point && k == key)
}
/// Returns `true` (consuming the arm) iff a crash is armed for exactly this
/// `point` and `key`.
pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {
+48 -48
View File
@@ -624,58 +624,58 @@ mod tests {
#[tokio::test]
async fn cleanup_dispatch_retains_resource_units_without_inheriting_disk_policy() {
use crate::disk::io_admission::{current_disk_io_context, test_resource_units, with_background_disk_io};
use crate::disk::io_admission::{current_disk_io_context, with_background_disk_io, with_background_resource_unit};
use rustfs_concurrency::adaptive::AdaptiveAdmission;
use std::num::NonZeroU16;
use tokio_util::sync::CancellationToken;
for physical_count in [1, 2] {
for mode in 0..4 {
let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
let worker_runtime = runtime.clone();
let (unit, gates) = test_resource_units(physical_count);
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let work = async move {
started_tx
.send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
.expect("report worker context");
release_rx.await.expect("release owned worker");
Ok::<_, super::super::error::DiskError>(())
};
let task = tokio::spawn(with_background_disk_io(
CancellationToken::new(),
with_optional_resource_unit(Some(unit), async move {
match mode {
0 => worker_runtime
.disk_operation(work, DiskAdmission::Request)
.await
.expect("disk operation"),
1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
2 => {
let reservation = worker_runtime.reserve().expect("cleanup reservation");
run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
}
_ => spawn(None, work)
.await
.expect("unisolated worker")
.expect("unisolated operation"),
for mode in 0..4 {
let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
let worker_runtime = runtime.clone();
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let work = async move {
started_tx
.send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
.expect("report worker context");
release_rx.await.expect("release owned worker");
Ok::<_, super::super::error::DiskError>(())
};
let task = tokio::spawn(with_background_disk_io(
CancellationToken::new(),
with_background_resource_unit(gate.acquire().await, async move {
match mode {
0 => worker_runtime
.disk_operation(work, DiskAdmission::Request)
.await
.expect("disk operation"),
1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
2 => {
let reservation = worker_runtime.reserve().expect("cleanup reservation");
run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
}
}),
));
let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
task.abort();
assert!(task.await.expect_err("waiter cancelled").is_cancelled());
let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
release_tx.send(()).expect("owned work survives cancellation");
deadline(async {
while gates.iter().any(|gate| gate.snapshot().active != 0) {
tokio::task::yield_now().await;
_ => spawn(None, work)
.await
.expect("unisolated worker")
.expect("unisolated operation"),
}
})
.await;
assert!(inherited, "mode {mode} lost resource ownership");
assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
assert_eq!(held, vec![1; physical_count], "mode {mode} released a live worker's unit");
}
}),
));
let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
task.abort();
assert!(task.await.expect_err("waiter cancelled").is_cancelled());
let held = gate.snapshot().active;
release_tx.send(()).expect("owned work survives cancellation");
deadline(async {
while gate.snapshot().active != 0 {
tokio::task::yield_now().await;
}
})
.await;
assert!(inherited, "mode {mode} lost resource ownership");
assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
assert_eq!(held, 1, "mode {mode} released a live worker's unit");
}
}
fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config {
+33 -745
View File
@@ -40,7 +40,6 @@ use std::sync::OnceLock;
use std::sync::atomic::AtomicBool;
use std::{
collections::HashMap,
io,
path::PathBuf,
pin::Pin,
sync::{
@@ -100,16 +99,6 @@ async fn observe_disk_operation<F: std::future::Future>(
}
}
// The health, admission and timeout layers above a disk operation each embed its
// future, and unoptimized builds copy it into every awaiting frame. Box it where
// it is created so those layers carry only a pointer: one allocation per operation.
fn boxed_disk_operation<F, Fut>(operation: F, io_permit: Option<crate::disk::io_admission::DiskIoPermit>) -> Pin<Box<Fut>>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
{
Box::pin(operation(io_permit))
}
struct SlowDiskWriter {
inner: crate::disk::FileWriter,
endpoint: Endpoint,
@@ -1585,8 +1574,6 @@ pub struct LocalDiskWrapper {
metrics: Arc<DiskHealthMetricEpoch>,
/// Per-drive foreground/background admission shared across wrapper reconnects.
io_admission: Arc<DiskIoAdmission>,
/// Physical leaves are refreshed by the node sampler, never by object I/O.
physical_admission: Arc<crate::disk::io_admission::PhysicalDiskBinding>,
/// Whether health checking is enabled
health_check: bool,
/// Cancellation token for monitoring tasks
@@ -1741,7 +1728,6 @@ impl LocalDiskWrapper {
health,
metrics,
io_admission,
physical_admission: Arc::new(crate::disk::io_admission::PhysicalDiskBinding::default()),
health_check: health_check && env_health_check,
cancel_token: CancellationToken::new(),
disk_id: Arc::new(RwLock::new(None)),
@@ -1759,10 +1745,6 @@ impl LocalDiskWrapper {
}
}
pub(crate) fn physical_admission(&self) -> Arc<crate::disk::io_admission::PhysicalDiskBinding> {
self.physical_admission.clone()
}
pub fn get_disk(&self) -> Arc<LocalDisk> {
self.disk.clone()
}
@@ -1804,33 +1786,6 @@ impl LocalDiskWrapper {
context
}
// Projection cannot wait: callers may already hold namespace or cache locks.
// Enter before identity probes, disk permits, or owned native jobs are created.
async fn with_native_resources<F: std::future::Future>(&self, operation: F) -> Result<F::Output> {
let Some(resources) = current_resource_unit() else {
if super::remote_io_admission::active() {
return Err(DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)));
}
return Ok(operation.await);
};
let resources = if let Some(permits) = resources.as_set() {
let unavailable = || DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock));
let plan = self.physical_admission.plan().ok_or_else(unavailable)?;
let native = plan
.capture_or_acquire_native(permits)
.map_err(|error| DiskError::Io(io::Error::new(io::ErrorKind::WouldBlock, error)))?;
if !plan.is_current() {
return Err(unavailable());
}
resources.with_permits(native)
} else {
resources
};
#[cfg(target_os = "linux")]
let resources = resources.with_mount_lease(self.disk.native_mount_lease());
Ok(with_optional_resource_unit(Some(resources), operation).await)
}
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
if let Some(context) = current_disk_io_context() {
if let Some(permit) = context.permit
@@ -2246,19 +2201,6 @@ impl LocalDiskWrapper {
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<Error>> {
let count = versions.len();
self.with_native_resources(self.delete_versions_admitted(volume, versions, opts, namespace_owner))
.await
.unwrap_or_else(|error| vec![Some(error); count])
}
async fn delete_versions_admitted(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<Error>> {
let version_count = versions.len();
self.metrics.record_operation_call("delete_versions");
@@ -2311,19 +2253,6 @@ impl LocalDiskWrapper {
versions: Vec<GuardedFileInfoVersions>,
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<Error>> {
let count = versions.len();
self.with_native_resources(self.delete_versions_guarded_admitted(volume, versions, opts, namespace_owner))
.await
.unwrap_or_else(|error| vec![Some(error); count])
}
async fn delete_versions_guarded_admitted(
&self,
volume: &str,
versions: Vec<GuardedFileInfoVersions>,
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<Error>> {
let version_count = versions.len();
self.metrics.record_operation_call("delete_versions");
@@ -2398,51 +2327,6 @@ impl LocalDiskWrapper {
.await
}
pub(crate) async fn try_publish_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: uuid::Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
self.track_disk_health_mutation(
"settle_delete_transaction",
DiskMetricMutation::Delete,
|| {
Box::pin(async {
self.disk
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
})
},
crate::disk::get_delete_recovery_timeout()?,
)
.await
}
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: uuid::Uuid,
journal: &crate::disk::DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<bool> {
self.track_disk_health_mutation(
"settle_delete_transaction",
DiskMetricMutation::Delete,
|| {
Box::pin(async {
self.disk
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
})
},
crate::disk::get_delete_recovery_timeout()?,
)
.await
}
/// Track disk health for an operation.
/// This method should wrap disk operations to ensure health checking.
pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
@@ -2567,28 +2451,6 @@ impl LocalDiskWrapper {
timeout_health_action: TimeoutHealthAction,
mutation: DiskMetricMutation,
) -> Result<T>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
self.with_native_resources(self.track_admitted_disk_operation(
op,
operation,
timeout_duration,
timeout_health_action,
mutation,
))
.await?
}
async fn track_admitted_disk_operation<T, F, Fut>(
&self,
op: &'static str,
operation: F,
timeout_duration: Duration,
timeout_health_action: TimeoutHealthAction,
mutation: DiskMetricMutation,
) -> Result<T>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
Fut: std::future::Future<Output = Result<T>>,
@@ -2621,14 +2483,13 @@ impl LocalDiskWrapper {
let _waiting_guard = self.health.waiting_guard();
let _metric_waiting_guard = self.metrics.waiting_guard();
let started = Instant::now();
let operation = observe_disk_operation(|| self.endpoint(), op, boxed_disk_operation(operation, io_permit));
let operation = observe_disk_operation(|| self.endpoint(), op, operation(io_permit));
if timeout_duration == Duration::ZERO {
let result = operation.await;
self.metrics.record_operation_latency(op, started.elapsed());
self.record_result_error_metrics(&result);
if result.is_ok() {
super::remote_io_admission::record_progress();
self.health.record_operation_success(&self.endpoint(), "operation_success");
self.metrics.record_mutation_success(mutation);
}
@@ -2643,7 +2504,6 @@ impl LocalDiskWrapper {
self.record_result_error_metrics(&operation_result);
// Log success; the waiting guard balances every exit path.
if operation_result.is_ok() {
super::remote_io_admission::record_progress();
self.health.record_operation_success(&self.endpoint(), "operation_success");
self.metrics.record_mutation_success(mutation);
}
@@ -2815,7 +2675,7 @@ impl DiskAPI for LocalDiskWrapper {
if self.health.is_faulty() {
return false;
}
let Ok(Some(disk_id)) = self.get_disk_id().await else {
let Ok(Some(disk_id)) = self.disk.get_disk_id().await else {
return false;
};
@@ -2848,7 +2708,7 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
self.with_native_resources(self.disk.get_disk_id()).await?
self.disk.get_disk_id().await
}
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
@@ -3280,29 +3140,25 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn read_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileReader> {
self.track_disk_health_with_op(
"read_file",
|| async {
let reader = self.disk.read_file(volume, path).await?;
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context()))
as crate::disk::FileReader)
},
get_max_timeout_duration(),
)
.await
let reader = self
.track_disk_health_with_op(
"read_file",
|| async { self.disk.read_file(volume, path).await },
get_max_timeout_duration(),
)
.await?;
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
}
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<crate::disk::FileReader> {
self.track_disk_health_with_op(
"read_file_stream",
|| async {
let reader = self.disk.read_file_stream(volume, path, offset, length).await?;
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context()))
as crate::disk::FileReader)
},
get_max_timeout_duration(),
)
.await
let reader = self
.track_disk_health_with_op(
"read_file_stream",
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await?;
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
}
async fn read_file_stream_chunks(
@@ -3312,21 +3168,20 @@ impl DiskAPI for LocalDiskWrapper {
offset: usize,
length: usize,
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
self.track_disk_health_with_op(
"read_file_stream_chunks",
|| async {
let reader = self.disk.read_file_stream_chunks(volume, path, offset, length).await?;
Ok(reader.map(|reader| {
Box::new(AdmissionReader::new(
BoxedChunkReader(reader),
self.io_admission.clone(),
self.disk_io_context(),
)) as rustfs_rio::ChunkReaderBox
}))
},
get_max_timeout_duration(),
)
.await
let reader = self
.track_disk_health_with_op(
"read_file_stream_chunks",
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await?;
Ok(reader.map(|reader| {
Box::new(AdmissionReader::new(
BoxedChunkReader(reader),
self.io_admission.clone(),
self.disk_io_context(),
)) as rustfs_rio::ChunkReaderBox
}))
}
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
@@ -4291,573 +4146,6 @@ mod tests {
);
}
#[tokio::test]
async fn local_disk_resource_identity_probes_keep_only_their_native_device_after_cancel() {
assert_native_identity_probe_cancellation(false).await;
}
#[tokio::test]
async fn remote_background_native_identity_cancellation_keeps_receiver_device() {
assert_native_identity_probe_cancellation(true).await;
}
async fn assert_native_identity_probe_cancellation(remote: bool) {
use rustfs_concurrency::adaptive::{
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
permit_set::AdaptivePermitSetOwner,
};
use std::num::NonZeroU16;
for read in [false, true] {
for online in [false, true] {
let dir = tempfile::tempdir().expect("format probe fixture");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let meta = dir.path().join(RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
.await
.expect("write format");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
{
let mut cache = disk.format_info.write().await;
cache.last_check = None;
if read {
cache.file_info = None;
}
}
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
disk.install_format_probe_hook(read, move || {
let _ = started_tx.send(());
let _ = release_rx.blocking_recv();
});
let wrapper = Arc::new(LocalDiskWrapper::new(disk, false));
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("catalog");
wrapper
.physical_admission
.set_test_shared_capacity(shared.clone(), vec![a.clone()])
.expect("native device A");
let node = NodeAdmission::new(NonZeroU16::MIN);
node.update(
&mut FeedbackController::new(),
Some(FeedbackObservation {
generation: 1,
max_concurrency: 1,
pressure: ResourcePressure::Unknown,
completed_units: None,
}),
);
let parent = (!remote).then(|| shared.try_acquire_with_node(&node).expect("planned devices"));
let task = tokio::spawn({
let wrapper = wrapper.clone();
let node = node.clone();
async move {
let operation = async {
if online {
assert!(wrapper.is_online().await);
} else {
wrapper.get_disk_id().await.expect("disk identity");
}
};
if remote {
crate::disk::with_remote_background_io(Some(crate::disk::RemoteBackgroundIo::new(node)), async {
crate::disk::remote_io_admission::admit(&wrapper.physical_admission).expect("RPC disk admission");
operation.await;
})
.await;
} else {
let parent = parent.expect("local operation admission");
crate::disk::with_background_resource_set(parent.resources().clone(), operation).await;
parent.complete();
}
}
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("native probe starts")
.expect("native signal");
task.abort();
assert!(task.await.expect_err("cancel waiter").is_cancelled());
let held = [node.snapshot().active, a.snapshot().active, b.snapshot().active];
let healthy_available = b.try_acquire().is_some();
release_tx.send(()).expect("release native probe before assertions");
tokio::time::timeout(Duration::from_secs(5), async {
while a.snapshot().active != 0 || b.snapshot().active != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("native probe releases its device");
assert_eq!(held, [0, 1, 0], "read={read}, online={online}");
assert!(healthy_available);
assert_eq!([node.completed(), a.completed(), b.completed()], [Some(0), Some(0), Some(0)]);
assert_eq!(wrapper.get_disk_id().await.expect("format remains readable"), Some(format.erasure.this));
assert!(wrapper.is_online().await);
}
}
}
#[tokio::test]
async fn local_disk_resource_projection_releases_unrelated_devices_after_native_cancellation() {
use crate::disk::io_admission::spawn_blocking_with_disk_io_permit;
use rustfs_concurrency::adaptive::{
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
permit_set::AdaptivePermitSetOwner,
};
use std::num::NonZeroU16;
for auxiliary in [false, true] {
let dir = tempfile::tempdir().expect("local disk fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(disk, false);
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()])
.expect("shared operation resources");
wrapper
.physical_admission
.set_test_shared_capacity(shared.clone(), vec![if auxiliary { second.clone() } else { first.clone() }])
.expect("only the first device backs this wrapper");
let node = NodeAdmission::new(NonZeroU16::MIN);
node.update(
&mut FeedbackController::new(),
Some(FeedbackObservation {
generation: 1,
max_concurrency: 1,
pressure: ResourcePressure::Unknown,
completed_units: None,
}),
);
let parent = if auxiliary {
shared
.select(std::slice::from_ref(&first))
.expect("initial object device")
.try_acquire_with_node(&node)
} else {
shared.try_acquire_with_node(&node)
}
.expect("initial object admission");
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
crate::disk::with_background_resource_set(parent.resources().clone(), async {
wrapper
.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
"read_all",
move |permit| async move {
spawn_blocking_with_disk_io_permit(permit, move || {
started_tx.send(()).expect("signal native start");
release_rx.blocking_recv().expect("release native I/O");
})
.await
.expect("native I/O joined");
Ok(())
},
Duration::ZERO,
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::None,
)
.await
.expect("wrapped I/O");
})
.await;
parent.complete();
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("native work starts")
.expect("start notification");
task.abort();
assert!(task.await.expect_err("cancel waiter").is_cancelled());
let held = [node.snapshot().active, first.snapshot().active, second.snapshot().active];
let healthy_available = if auxiliary {
first.try_acquire().is_some()
} else {
second.try_acquire().is_some()
};
release_tx.send(()).expect("finish native work before asserting");
tokio::time::timeout(Duration::from_secs(5), async {
while first.snapshot().active != 0 || second.snapshot().active != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("native resource return");
assert_eq!(
held,
if auxiliary { [0, 0, 1] } else { [0, 1, 0] },
"only the actual native device remains charged"
);
assert!(healthy_available, "healthy work must resume before native I/O finishes");
assert_eq!(node.completed(), Some(0), "cancelled work is not progress");
assert_eq!([first.completed(), second.completed()], [Some(0), Some(0)]);
}
}
#[tokio::test]
async fn local_disk_resource_projection_escaped_readers_keep_only_their_device() {
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
use std::num::NonZeroU16;
use tokio::io::AsyncReadExt;
let dir = tempfile::tempdir().expect("reader fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(disk, false);
tokio::fs::create_dir_all(dir.path().join("bucket"))
.await
.expect("bucket directory");
let bytes = b"physical disk reader payload";
tokio::fs::write(dir.path().join("bucket/object"), bytes)
.await
.expect("reader bytes");
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()]).expect("parent resources");
wrapper
.physical_admission
.set_test_shared_capacity(shared.clone(), vec![first.clone()])
.expect("reader device");
for stream in [false, true] {
let a = first.try_acquire().expect("first device");
let b = second.try_acquire().expect("second device");
let parent = shared.capture(&[&a, &b]).expect("admitted parent");
drop((a, b));
let mut reader = crate::disk::with_background_resource_set(parent, async {
if stream {
wrapper.read_file_stream("bucket", "object", 0, bytes.len()).await
} else {
wrapper.read_file("bucket", "object").await
}
})
.await
.expect("reader escapes the resource scope");
assert_eq!([first.snapshot().active, second.snapshot().active], [1, 0]);
assert!(second.try_acquire().is_some(), "unrelated work is available while the reader lives");
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("consume escaped reader");
assert_eq!(actual, bytes);
drop(reader);
assert_eq!(first.snapshot().active, 0);
}
}
#[cfg(unix)]
#[test]
fn local_disk_resource_projection_cancelled_stream_read_retains_native_device() {
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
use std::num::NonZeroU16;
use tokio::io::{AsyncRead, ReadBuf};
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.max_blocking_threads(1)
.build()
.expect("one-thread native read pool");
runtime.block_on(async {
let dir = tempfile::tempdir().expect("native reader fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(disk, false);
std::fs::create_dir(dir.path().join("bucket")).expect("bucket directory");
let bytes = b"physical disk native reader payload";
std::fs::write(dir.path().join("bucket/object"), bytes).expect("reader bytes");
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared =
AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()]).expect("parent resources");
wrapper
.physical_admission
.set_test_shared_capacity(shared.clone(), vec![first.clone()])
.expect("reader device");
let mut observations = Vec::new();
for stream in [false, true] {
let a = first.try_acquire().expect("first device");
let b = second.try_acquire().expect("second device");
let parent = shared.capture(&[&a, &b]).expect("admitted parent");
drop((a, b));
let mut reader = crate::disk::with_background_resource_set(
parent,
crate::disk::with_background_disk_io(CancellationToken::new(), async {
if stream {
wrapper.read_file_stream("bucket", "object", 3, bytes.len() - 3).await
} else {
wrapper.read_file("bucket", "object").await
}
}),
)
.await
.expect("background reader escapes its parent");
assert_eq!([first.snapshot().active, second.snapshot().active], [1, 0]);
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let blocker = tokio::task::spawn_blocking(move || {
let _ = started_tx.send(());
let _ = release_rx.recv();
});
started_rx.await.expect("native pool occupied after open");
let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
let mut data = [0; 1];
let mut pending_tx = Some(pending_tx);
std::future::poll_fn(|cx| {
let mut buffer = ReadBuf::new(&mut data);
let result = Pin::new(&mut reader).poll_read(cx, &mut buffer);
if result.is_pending()
&& let Some(signal) = pending_tx.take()
{
let _ = signal.send(());
}
result
})
.await
});
tokio::time::timeout(Duration::from_secs(5), pending_rx)
.await
.expect("read queues behind native worker")
.expect("pending read signal");
let admitted = wrapper.background_io_active_for_tests();
task.abort();
assert!(task.await.expect_err("cancel pending read").is_cancelled());
let held = [first.snapshot().active, second.snapshot().active];
release_tx.send(()).expect("release native pool before asserting");
blocker.await.expect("blocking worker released");
tokio::task::spawn_blocking(|| ())
.await
.expect("drain the single native queue");
assert_eq!([first.snapshot().active, second.snapshot().active], [0, 0]);
observations.push((stream, admitted, held));
}
for (stream, admitted, held) in observations {
assert!(admitted, "stream={stream}: cancellation follows disk admission, not its wait");
assert_eq!(held, [1, 0], "stream={stream}: only the pending native read's device stays charged");
}
});
}
#[cfg(unix)]
#[tokio::test]
async fn projected_stream_memory_survives_scope_exit_and_hot_shrink() {
use crate::disk::io_admission::{BackgroundMemoryBudget, ResourceIoUnit, with_optional_resource_unit};
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
use std::num::NonZeroU16;
use tokio::io::AsyncReadExt;
let dir = tempfile::tempdir().expect("memory admission fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(disk, false);
std::fs::create_dir(dir.path().join("bucket")).expect("bucket directory");
let bytes = b"0123456789abcdefghijklmnopqrstuv";
std::fs::write(dir.path().join("bucket/object"), bytes).expect("reader bytes");
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()]).expect("parent resources");
wrapper
.physical_admission
.set_test_shared_capacity(shared.clone(), vec![first.clone()])
.expect("selected device");
let memory = BackgroundMemoryBudget::new();
for stream in [false, true] {
memory.set_sampled_limit(4, Duration::from_secs(30));
let a = first.try_acquire().expect("first device");
let b = second.try_acquire().expect("second device");
let parent =
ResourceIoUnit::from(shared.capture(&[&a, &b]).expect("admitted parent")).with_memory_budget(memory.clone());
drop((a, b));
let mut reader = with_optional_resource_unit(Some(parent), async {
if stream {
wrapper.read_file_stream("bucket", "object", 0, bytes.len()).await
} else {
wrapper.read_file("bucket", "object").await
}
})
.await
.expect("reader escapes its resource scope");
assert_eq!(memory.snapshot().used, 0, "opening does not allocate a read buffer");
let mut prefix = [0; 4];
reader.read_exact(&mut prefix).await.expect("initial read");
assert_eq!(&prefix, &bytes[..4]);
assert_eq!(memory.snapshot().used, 4, "projection retains the same memory owner");
assert_eq!([first.snapshot().active, second.snapshot().active], [1, 0]);
memory.set_sampled_limit(0, Duration::ZERO);
let mut reused = [0; 2];
reader
.read_exact(&mut reused)
.await
.expect("already allocated storage remains usable");
assert_eq!(&reused, &bytes[4..6]);
let mut next = [99; 8];
let error = reader.read(&mut next).await.expect_err("new storage is blocked after shrink");
assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock);
assert_eq!(next, [99; 8], "denied allocation exposes no stale bytes");
assert_eq!(memory.snapshot().used, 4);
memory.set_sampled_limit(12, Duration::from_secs(30));
reader
.read_exact(&mut next)
.await
.expect("growth includes old and replacement storage");
assert_eq!(&next, &bytes[6..14], "denial did not advance the file");
assert_eq!(memory.snapshot().used, 8);
drop(reader);
assert_eq!(memory.snapshot().used, 0);
assert_eq!([first.snapshot().active, second.snapshot().active], [0, 0]);
}
}
#[tokio::test]
async fn local_disk_resource_projection_auxiliary_metadata_retries_without_parking_or_false_progress() {
use rustfs_concurrency::adaptive::{
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
permit_set::AdaptivePermitSetOwner,
};
use std::num::NonZeroU16;
let dir = tempfile::tempdir().expect("metadata fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(disk, false);
tokio::fs::create_dir_all(dir.path().join("bucket"))
.await
.expect("metadata bucket");
tokio::fs::write(dir.path().join("bucket/metadata"), b"auxiliary metadata")
.await
.expect("metadata bytes");
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("topology");
wrapper
.physical_admission
.set_test_shared_capacity(shared.clone(), vec![b.clone()])
.expect("metadata on B");
let node = NodeAdmission::new(NonZeroU16::MIN);
node.update(
&mut FeedbackController::new(),
Some(FeedbackObservation {
generation: 1,
max_concurrency: 1,
pressure: ResourcePressure::Unknown,
completed_units: None,
}),
);
let parent = shared
.select(std::slice::from_ref(&a))
.expect("object on A")
.try_acquire_with_node(&node)
.expect("object admission");
crate::disk::with_background_resource_set(parent.resources().clone(), async {
let mut read = Box::pin(wrapper.read_all("bucket", "metadata"));
let rejected = std::future::poll_fn(|cx| Poll::Ready(read.as_mut().poll(cx))).await;
assert!(
matches!(rejected, Poll::Ready(Err(DiskError::Io(error))) if error.kind() == io::ErrorKind::WouldBlock),
"auxiliary admission must return immediately under storage locks"
);
drop(read);
assert!(
matches!(wrapper.get_disk_id().await, Err(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock)
);
assert!(!wrapper.is_online().await, "identity checks cannot bypass closed physical capacity");
assert_eq!([a.snapshot().active, b.snapshot().active, node.snapshot().active], [1, 0, 1]);
assert_eq!([a.completed(), b.completed(), node.completed()], [Some(0), Some(0), Some(0)]);
b.set_limit(1);
for _ in 0..2 {
assert_eq!(
wrapper.read_all("bucket", "metadata").await.expect("retry metadata read"),
Bytes::from_static(b"auxiliary metadata")
);
assert_eq!(b.snapshot().active, 1, "repeated I/O reuses the parent's physical unit");
}
})
.await;
assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 0);
assert_eq!([a.completed(), b.completed(), node.completed()], [Some(0), Some(0), Some(0)]);
parent.complete();
assert_eq!([a.completed(), b.completed(), node.completed()], [Some(1), Some(1), Some(1)]);
assert_eq!([a.snapshot().active, b.snapshot().active, node.snapshot().active], [0, 0, 0]);
}
#[tokio::test]
async fn local_disk_resource_projection_defers_missing_or_foreign_owners_before_io() {
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
use std::num::NonZeroU16;
let dir = tempfile::tempdir().expect("unavailable resource fixture");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
let wrapper = LocalDiskWrapper::new(disk, false);
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![gate.clone()]).expect("parent");
let permit = gate.try_acquire().expect("parent device");
let parent = shared.capture(&[&permit]).expect("parent ownership");
drop(permit);
for foreign in [false, true] {
if foreign {
wrapper
.physical_admission
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
.expect("replaced device");
}
crate::disk::with_background_resource_set(parent.clone(), async {
let polled = std::cell::Cell::new(false);
let result = wrapper
.track_disk_health_with_op(
"read_all",
|| async {
polled.set(true);
Ok(())
},
Duration::ZERO,
)
.await;
assert!(matches!(result, Err(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock));
assert!(!polled.get(), "unavailable ownership cannot start native work");
let group = FileInfoVersions::default();
let plain = wrapper
.delete_versions("bucket", vec![group.clone(); 2], DeleteOptions::default())
.await;
let guarded = wrapper
.delete_versions_guarded_with_namespace_owner(
"bucket",
vec![
GuardedFileInfoVersions {
group,
expected_current: crate::disk::CurrentDeleteGeneration::Absent
};
2
],
DeleteOptions::default(),
None,
)
.await;
for errors in [plain, guarded] {
assert_eq!(errors.len(), 2);
assert!(errors.iter().all(|error| matches!(error,
Some(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock)));
}
})
.await;
assert_eq!(wrapper.health.waiting_count(), 0);
assert_eq!(
wrapper.metrics_snapshot().total_errors_availability,
0,
"resource deferral is not a failed drive"
);
assert_eq!(gate.snapshot().active, 1, "rejection does not refund parent ownership");
}
}
async fn assert_multipart_recovery_waiter_preserves_physical_owner(wait_timeout: Duration) {
use crate::disk::multipart_source::{
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartRecoveryRound, MultipartSourceOperation, MultipartSourcePhase,
+62 -132
View File
@@ -16,15 +16,40 @@ use std::{
collections::HashMap,
fs::Metadata,
path::{Path, PathBuf},
sync::Mutex,
sync::{Arc, Mutex, OnceLock},
time::{Duration, Instant},
};
use tokio::{fs::File, io};
use tokio::{
fs::{self, File},
io,
};
async fn native_io<T: Send + 'static>(operation: impl FnOnce() -> io::Result<T> + Send + 'static) -> io::Result<T> {
super::io_admission::spawn_blocking_with_resource_ownership(operation)
.await
.map_err(|_| io::Error::other("background task failed"))?
static READONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
static WRITEONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
static READWRITE_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
fn get_readonly_options() -> &'static Arc<fs::OpenOptions> {
READONLY_OPTIONS.get_or_init(|| {
let mut opts = fs::OpenOptions::new();
opts.read(true);
Arc::new(opts)
})
}
fn get_writeonly_options() -> &'static Arc<fs::OpenOptions> {
WRITEONLY_OPTIONS.get_or_init(|| {
let mut opts = fs::OpenOptions::new();
opts.write(true);
Arc::new(opts)
})
}
fn get_readwrite_options() -> &'static Arc<fs::OpenOptions> {
READWRITE_OPTIONS.get_or_init(|| {
let mut opts = fs::OpenOptions::new();
opts.read(true).write(true);
Arc::new(opts)
})
}
#[cfg(not(windows))]
@@ -92,32 +117,33 @@ pub const O_APPEND: FileMode = 0x00400;
// create_new: bool,
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
let path = path.as_ref().to_path_buf();
native_io(move || {
let mut options = std::fs::OpenOptions::new();
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
O_WRONLY => options.write(true),
O_RDWR => options.read(true).write(true),
_ => options.read(true),
let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
O_RDONLY => get_readonly_options(),
O_WRONLY => get_writeonly_options(),
O_RDWR => get_readwrite_options(),
_ => get_readonly_options(),
};
if (mode & (O_CREATE | O_APPEND | O_TRUNC)) != 0 {
let mut opts = (**base_opts).clone();
if mode & O_CREATE != 0 {
opts.create(true);
}
.create(mode & O_CREATE != 0)
.append(mode & O_APPEND != 0)
.truncate(mode & O_TRUNC != 0);
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
if mode == O_RDONLY {
super::os::prepared_publication_test_hooks::run(
super::os::prepared_publication_test_hooks::Stage::NativeReadOpen,
&path,
);
if mode & O_APPEND != 0 {
opts.append(true);
}
options.open(path).map(File::from_std)
})
.await
if mode & O_TRUNC != 0 {
opts.truncate(true);
}
opts.open(path.as_ref()).await
} else {
base_opts.open(path.as_ref()).await
}
}
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::metadata(path).map(|_| ())).await
fs::metadata(path).await?;
Ok(())
}
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
@@ -126,8 +152,7 @@ pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
}
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::symlink_metadata(path)).await
fs::symlink_metadata(path).await
}
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
@@ -145,7 +170,7 @@ pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref().to_path_buf();
native_io(move || create_dir_all_std(&path)).await
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
}
fn is_dir_error(e: &io::Error) -> bool {
@@ -158,13 +183,9 @@ fn is_dir_error(e: &io::Error) -> bool {
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
// Try remove_file first; fall back to remove_dir if it's a directory
let file_path = path.as_ref().to_path_buf();
match native_io(move || std::fs::remove_file(file_path)).await {
match fs::remove_file(path.as_ref()).await {
Ok(()) => Ok(()),
Err(e) if is_dir_error(&e) => {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::remove_dir(path)).await
}
Err(e) if is_dir_error(&e) => fs::remove_dir(path.as_ref()).await,
Err(e) => Err(e),
}
}
@@ -172,13 +193,9 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
// Try remove_file first; fall back to remove_dir_all if it's a directory
let file_path = path.as_ref().to_path_buf();
match native_io(move || std::fs::remove_file(file_path)).await {
match fs::remove_file(path.as_ref()).await {
Ok(()) => Ok(()),
Err(e) if is_dir_error(&e) => {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::remove_dir_all(path)).await
}
Err(e) if is_dir_error(&e) => fs::remove_dir_all(path.as_ref()).await,
Err(e) => Err(e),
}
}
@@ -205,14 +222,11 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
#[cfg(test)]
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::create_dir(path)).await
fs::create_dir(path.as_ref()).await
}
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
let from = from.as_ref().to_path_buf();
let to = to.as_ref().to_path_buf();
native_io(move || std::fs::rename(from, to)).await
fs::rename(from, to).await
}
#[cfg(any(not(windows), test))]
@@ -275,7 +289,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
}
let path_buf = path.to_path_buf();
let result = access(&path_buf).await;
let result = fs::metadata(&path_buf).await;
match &result {
Ok(_) => BUCKET_EXISTENCE_CACHE.record(path_buf, true),
@@ -290,7 +304,6 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
}
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
BUCKET_EXISTENCE_CACHE.invalidate(path.as_ref());
}
@@ -300,89 +313,6 @@ mod tests {
use tempfile::TempDir;
use tokio::io::AsyncWriteExt;
#[test]
fn cancelled_filesystem_jobs_retain_physical_resources() {
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.max_blocking_threads(1)
.build()
.expect("single native worker");
runtime.block_on(async {
let mut observations = Vec::new();
for count in 1..=2 {
for operation in [
"access",
"cached_access",
"lstat",
"mkdir",
"make_dir_all",
"remove",
"remove_all",
"rename",
"open",
] {
let dir = TempDir::new().expect("filesystem ownership fixture");
let source = dir.path().join("source");
let target = dir.path().join("target");
match operation {
"mkdir" | "make_dir_all" | "open" => {}
"remove_all" => {
std::fs::create_dir(&source).expect("recursive removal directory");
std::fs::write(source.join("child"), b"payload").expect("recursive removal child");
}
_ => std::fs::write(&source, b"payload").expect("existing file"),
}
let (resources, gates) = test_resource_units(count);
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
let blocker = tokio::task::spawn_blocking(move || {
let _ = started_tx.send(());
let _ = release_rx.recv();
});
started_rx.await.expect("native worker occupied");
let mut future = Box::pin(with_optional_resource_unit(Some(resources), async {
match operation {
"access" => access(&source).await,
"cached_access" => cached_access(&source).await,
"lstat" => lstat(&source).await.map(|_| ()),
"mkdir" => mkdir(&source).await,
"make_dir_all" => make_dir_all(source.join("child")).await,
"remove" => remove(&source).await,
"remove_all" => remove_all(&source).await,
"rename" => rename(&source, &target).await,
"open" => open_file(&source, O_CREATE | O_WRONLY).await.map(|_| ()),
_ => unreachable!("fixed operation list"),
}
}));
assert!(futures::poll!(&mut future).is_pending(), "{operation}: native job queued");
drop(future);
let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
release_tx.send(()).expect("release native worker before assertions");
blocker.await.expect("native blocker completed");
tokio::task::spawn_blocking(|| ()).await.expect("native queue drained");
assert!(gates.iter().all(|gate| gate.snapshot().active == 0));
match operation {
"mkdir" => assert!(source.is_dir()),
"make_dir_all" => assert!(source.join("child").is_dir()),
"open" => assert!(source.is_file()),
"remove" => assert!(!source.exists()),
"rename" => {
assert!(!source.exists());
assert_eq!(std::fs::read(&target).expect("renamed file"), b"payload");
}
_ => {}
}
observations.push((count, operation, held));
}
}
for (count, operation, held) in observations {
assert_eq!(held, vec![1; count], "{operation}: queued native job keeps its physical resources");
}
});
}
#[tokio::test]
async fn test_file_mode_constants() {
assert_eq!(O_RDONLY, 0x00000);
+15 -218
View File
@@ -14,18 +14,6 @@
//! Per-local-disk foreground-priority admission for scanner I/O.
pub(crate) mod memory;
mod physical_identity;
mod physical_owners;
mod physical_sampler;
pub use memory::BackgroundMemoryBudget;
pub(crate) use physical_sampler::PhysicalHealBinding;
pub use physical_sampler::{
PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler, PhysicalResourcePlan,
};
use std::collections::VecDeque;
use std::future::Future;
use std::io;
@@ -37,118 +25,18 @@ use std::time::Duration;
use metrics::{counter, gauge, histogram};
use rustfs_concurrency::adaptive::AdaptivePermit;
use rustfs_concurrency::adaptive::permit_set::AdaptivePermitSet;
use rustfs_rio::ChunkReader;
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::Notify;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
pub(crate) mod remote;
pub use remote::{RemoteBackgroundIo, with_remote_background_io};
const BACKGROUND_FOREGROUND_BURST: u32 = 8;
const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
tokio::task_local! {
static DISK_IO_CONTEXT: DiskIoContext;
static RESOURCE_IO_UNIT: ResourceIoUnit;
}
/// Native I/O retains the physical subset selected before storage work. A
/// layered disk may touch several leaves; unrelated devices stay outside it.
#[derive(Clone, Debug)]
pub(crate) struct ResourceIoUnit {
permits: ResourceIoPermits,
memory_budget: Option<Arc<BackgroundMemoryBudget>>,
#[cfg(target_os = "linux")]
_mount_lease: Option<Arc<std::fs::File>>,
}
#[derive(Debug)]
enum ResourceIoPermits {
Single(AdaptivePermit),
Set(AdaptivePermitSet),
}
impl Clone for ResourceIoPermits {
fn clone(&self) -> Self {
match self {
Self::Single(permit) => Self::Single(permit.clone()),
Self::Set(permits) => Self::Set(permits.clone()),
}
}
}
impl From<AdaptivePermit> for ResourceIoUnit {
fn from(permit: AdaptivePermit) -> Self {
Self {
permits: ResourceIoPermits::Single(permit),
memory_budget: BackgroundMemoryBudget::current(),
#[cfg(target_os = "linux")]
_mount_lease: None,
}
}
}
impl From<AdaptivePermitSet> for ResourceIoUnit {
fn from(permits: AdaptivePermitSet) -> Self {
Self {
permits: ResourceIoPermits::Set(permits),
memory_budget: BackgroundMemoryBudget::current(),
#[cfg(target_os = "linux")]
_mount_lease: None,
}
}
}
impl ResourceIoUnit {
pub(crate) fn memory_budget(&self) -> Option<Arc<BackgroundMemoryBudget>> {
self.memory_budget.clone()
}
pub(crate) fn with_permits(mut self, permits: AdaptivePermitSet) -> Self {
self.permits = ResourceIoPermits::Set(permits);
self
}
#[cfg(test)]
pub(crate) fn with_memory_budget(mut self, budget: Arc<BackgroundMemoryBudget>) -> Self {
self.memory_budget = Some(budget);
self
}
pub(crate) fn as_set(&self) -> Option<&AdaptivePermitSet> {
match &self.permits {
ResourceIoPermits::Single(_) => None,
ResourceIoPermits::Set(permits) => Some(permits),
}
}
#[cfg(target_os = "linux")]
pub(crate) fn with_mount_lease(mut self, mount_lease: Arc<std::fs::File>) -> Self {
// The selected disk's proc-fd paths must survive native jobs after its
// async wrapper retires. Clones carry the same pinned mount generation.
self._mount_lease = Some(mount_lease);
self
}
}
#[cfg(test)]
pub(crate) fn test_resource_units(count: usize) -> (ResourceIoUnit, Vec<Arc<rustfs_concurrency::adaptive::AdaptiveAdmission>>) {
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
use std::num::NonZeroU16;
assert!((1..=2).contains(&count), "test single and layered disks");
let gates: Vec<_> = (0..count).map(|_| AdaptiveAdmission::new(NonZeroU16::MIN, 1)).collect();
let mut admitted: Vec<_> = gates.iter().map(|gate| gate.try_acquire().expect("test resource")).collect();
let unit = if count == 1 {
ResourceIoUnit::from(admitted.pop().expect("single resource"))
} else {
let owner = AdaptivePermitSetOwner::new(NonZeroU16::MIN, gates.clone()).expect("layered resource");
ResourceIoUnit::from(owner.capture(&admitted.iter().collect::<Vec<_>>()).expect("admitted subset"))
};
(unit, gates)
static RESOURCE_IO_UNIT: AdaptivePermit;
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -163,7 +51,7 @@ pub(crate) struct DiskIoContext {
pub(crate) cancellation: CancellationToken,
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
pub(crate) permit: Option<DiskIoPermit>,
pub(crate) resource_unit: Option<ResourceIoUnit>,
pub(crate) resource_unit: Option<AdaptivePermit>,
}
/// Mark disk I/O issued by `future` as background work for local-disk admission.
@@ -190,46 +78,24 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
/// of a stalled native operation. Escaping buffers need their own memory leases.
#[doc(hidden)]
pub async fn with_background_resource_unit<F: Future>(unit: AdaptivePermit, future: F) -> F::Output {
RESOURCE_IO_UNIT.scope(ResourceIoUnit::from(unit), future).await
RESOURCE_IO_UNIT.scope(unit, future).await
}
/// Retain only the previously admitted physical leaves touched by this I/O.
/// Projection of owned units never waits and may run under storage locks.
/// Acquire the planned resources before locking; auxiliary leaves discovered
/// later must use nonblocking admission. Do not pass unrelated parent resources
/// to a native job that touches only some of its disks.
#[doc(hidden)]
pub async fn with_background_resource_set<F: Future>(units: AdaptivePermitSet, future: F) -> F::Output {
RESOURCE_IO_UNIT.scope(ResourceIoUnit::from(units), future).await
}
pub(crate) fn current_resource_unit() -> Option<ResourceIoUnit> {
RESOURCE_IO_UNIT
.try_with(Clone::clone)
.ok()
.or_else(|| {
DISK_IO_CONTEXT
.try_with(|context| context.resource_unit.clone())
.ok()
.flatten()
})
.or_else(remote::resources)
}
pub(crate) fn has_resource_io_scope() -> bool {
RESOURCE_IO_UNIT.try_with(|_| ()).is_ok()
|| DISK_IO_CONTEXT
.try_with(|context| context.resource_unit.is_some())
.unwrap_or(false)
|| remote::active()
pub(crate) fn current_resource_unit() -> Option<AdaptivePermit> {
RESOURCE_IO_UNIT.try_with(Clone::clone).ok().or_else(|| {
DISK_IO_CONTEXT
.try_with(|context| context.resource_unit.clone())
.ok()
.flatten()
})
}
/// Transfer only resource ownership into an existing spawned task. Disk
/// classification/permits are not inherited: changing those can introduce a
/// wait behind a disk permit still held by the parent.
pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<ResourceIoUnit>, future: F) -> F::Output {
pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<AdaptivePermit>, future: F) -> F::Output {
match unit {
Some(unit) => RESOURCE_IO_UNIT.scope(unit, future).await,
Some(unit) => with_background_resource_unit(unit, future).await,
None => future.await,
}
}
@@ -367,7 +233,7 @@ impl DiskIoAdmission {
self.foreground_with_resource(current_resource_unit())
}
fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<ResourceIoUnit>) -> DiskIoPermit {
fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<AdaptivePermit>) -> DiskIoPermit {
self.foreground_active.fetch_add(1, Ordering::AcqRel);
if self.background_waiters.load(Ordering::Acquire) > 0 {
self.record_state();
@@ -388,7 +254,7 @@ impl DiskIoAdmission {
async fn background_with_resource(
self: &Arc<Self>,
cancellation: &CancellationToken,
resource_unit: Option<ResourceIoUnit>,
resource_unit: Option<AdaptivePermit>,
) -> std::io::Result<DiskIoPermit> {
let started_at = Instant::now();
let ticket = {
@@ -563,7 +429,7 @@ impl DiskIoPermit {
struct DiskIoPermitLease {
admission: Arc<DiskIoAdmission>,
class: DiskIoClass,
resource_unit: Option<ResourceIoUnit>,
resource_unit: Option<AdaptivePermit>,
}
pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
@@ -584,18 +450,6 @@ where
})
}
/// Run native filesystem work with the current resource ownership. Cancelling
/// the async waiter cannot refund a unit while the native operation still runs.
/// This does not acquire capacity or change foreground/background policy.
#[doc(hidden)]
pub fn spawn_blocking_with_resource_ownership<T, F>(operation: F) -> tokio::task::JoinHandle<T>
where
T: Send + 'static,
F: FnOnce() -> T + Send + 'static,
{
spawn_blocking_with_disk_io_permit(None, operation)
}
type AdmissionWait = Pin<Box<dyn Future<Output = io::Result<DiskIoPermit>> + Send>>;
impl Drop for DiskIoPermitLease {
@@ -741,63 +595,6 @@ mod tests {
use std::num::NonZeroU16;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[tokio::test]
async fn physical_subset_survives_native_cancellation_without_retaining_unrelated_disks() {
use rustfs_concurrency::adaptive::permit_set::AdaptivePermitSetOwner;
for disk_permit in [false, true] {
let maximum = NonZeroU16::new(1).expect("one physical unit");
let first = AdaptiveAdmission::new(maximum, 1);
let second = AdaptiveAdmission::new(maximum, 1);
let unrelated = AdaptiveAdmission::new(maximum, 1);
let owner = AdaptivePermitSetOwner::new(maximum, vec![first.clone(), second.clone()]).expect("layered disk");
let a = first.try_acquire().expect("first backing device");
let b = second.try_acquire().expect("second backing device");
let healthy = unrelated.try_acquire().expect("another disk in the parent operation");
let subset = owner.capture(&[&a, &b]).expect("capture before storage work");
drop((a, b, owner));
let disk = DiskIoAdmission::new("physical-subset-native");
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
let _healthy = healthy;
with_background_resource_set(subset, async {
let permit = if disk_permit {
Some(disk.background(&CancellationToken::new()).await.expect("logical disk permit"))
} else {
None
};
spawn_blocking_with_disk_io_permit(permit, move || {
started_tx.send(()).expect("native I/O started");
release_rx.blocking_recv().expect("release native I/O");
})
.await
.expect("native I/O completed");
})
.await;
});
started_rx.await.expect("native worker retains subset");
task.abort();
assert!(task.await.expect_err("cancel async waiter").is_cancelled());
let active = [first.snapshot().active, second.snapshot().active, unrelated.snapshot().active];
let healthy_available = unrelated.try_acquire().is_some();
first.set_limit(0);
second.set_limit(0);
release_tx.send(()).expect("finish native work");
tokio::time::timeout(Duration::from_secs(5), async {
while first.snapshot().active != 0 || second.snapshot().active != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("both backing devices are returned after native completion");
assert_eq!(active, [1, 1, 0], "only the native I/O's physical subset remains charged");
assert!(healthy_available, "cancellation frees the independent disk immediately");
assert!(first.try_acquire().is_none());
assert!(second.try_acquire().is_none(), "old completions cannot reopen shrunken owners");
}
}
#[tokio::test]
async fn resource_unit_outlives_cancelled_scope_until_native_io_finishes() {
for permit_mode in 0..4 {
@@ -1,547 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use bytes::Bytes;
use std::io;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use tokio::time::Instant;
static BACKGROUND_MEMORY: OnceLock<Arc<BackgroundMemoryBudget>> = OnceLock::new();
/// Process-wide payload storage retained by admitted background I/O. The
/// existing resource sampler publishes limits; this owner starts no workers.
#[derive(Debug)]
pub struct BackgroundMemoryBudget {
// This short critical section only changes counters and a deadline. It
// never allocates, calls storage, or acquires another lock while held.
state: Mutex<State>,
}
#[derive(Debug)]
struct State {
limit: u64,
used: u64,
valid_until: Instant,
}
/// A coherent view of the leased cap and retained payload storage.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BackgroundMemorySnapshot {
/// Zero when the latest resource observation has expired.
pub limit: u64,
/// Existing storage remains charged even after the cap shrinks below it.
pub used: u64,
}
impl BackgroundMemoryBudget {
/// Start closed until the existing resource sampler publishes a limit.
pub fn new() -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(State {
limit: 0,
used: 0,
valid_until: Instant::now(),
}),
})
}
/// Install the sole process owner before starting adaptive background work.
/// Returns false if an owner was already installed.
pub fn install(self: &Arc<Self>) -> bool {
BACKGROUND_MEMORY.set(self.clone()).is_ok()
}
/// Return only the installed process memory policy; no node or disk ownership.
/// No policy is installed in the default, non-adaptive startup path.
pub fn current() -> Option<Arc<Self>> {
BACKGROUND_MEMORY.get().cloned()
}
/// Update admission without revoking storage retained by native tasks or
/// readers. Missing observations must publish zero capacity.
pub fn set_sampled_limit(&self, limit: u64, valid_for: Duration) {
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
state.limit = limit;
let now = Instant::now();
state.valid_until = now.checked_add(valid_for).unwrap_or(now);
}
/// Read the cap and retained byte count under the same short lock.
pub fn snapshot(&self) -> BackgroundMemorySnapshot {
let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
BackgroundMemorySnapshot {
limit: if Instant::now() < state.valid_until { state.limit } else { 0 },
used: state.used,
}
}
pub(crate) fn reserve(self: &Arc<Self>, bytes: usize) -> io::Result<Reservation> {
let bytes = u64::try_from(bytes).map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let used = state
.used
.checked_add(bytes)
.ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted))?;
if bytes != 0 && (Instant::now() >= state.valid_until || used > state.limit) {
return Err(io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted));
}
state.used = used;
Ok(Reservation {
owner: self.clone(),
bytes,
})
}
}
#[derive(Debug)]
struct BackgroundMemoryExhausted;
impl std::fmt::Display for BackgroundMemoryExhausted {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("background buffer memory budget exhausted")
}
}
impl std::error::Error for BackgroundMemoryExhausted {}
pub(crate) fn is_memory_pressure(error: &io::Error) -> bool {
error.kind() == io::ErrorKind::OutOfMemory || error.get_ref().is_some_and(|inner| inner.is::<BackgroundMemoryExhausted>())
}
pub(crate) struct Reservation {
owner: Arc<BackgroundMemoryBudget>,
bytes: u64,
}
impl Reservation {
pub(crate) fn belongs_to(&self, budget: &Arc<BackgroundMemoryBudget>) -> bool {
Arc::ptr_eq(&self.owner, budget)
}
pub(crate) fn wrap(bytes: Bytes, reservation: Option<Self>) -> Bytes {
if bytes.is_empty() {
drop(bytes);
drop(reservation);
return Bytes::new();
}
match reservation {
Some(reservation) => Bytes::from_owner(BackgroundBytes {
bytes,
_reservation: reservation,
}),
None => bytes,
}
}
}
struct BackgroundBytes {
// Free the payload before refunding it, including after the last slice.
bytes: Bytes,
_reservation: Reservation,
}
impl AsRef<[u8]> for BackgroundBytes {
fn as_ref(&self) -> &[u8] {
&self.bytes
}
}
impl Drop for Reservation {
fn drop(&mut self) {
self.owner
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.used -= self.bytes;
}
}
/// Reusable initialized payload storage. The reservation is embedded in the
/// existing native job state, with no per-read owner allocation or payload copy.
pub(crate) struct BackgroundBuffer {
// Field order releases the allocation before refunding its reservation.
bytes: Box<[u8]>,
reservation: Option<Reservation>,
budget: Option<Arc<BackgroundMemoryBudget>>,
}
impl BackgroundBuffer {
pub(crate) fn new(budget: Option<Arc<BackgroundMemoryBudget>>) -> Self {
Self {
bytes: Box::default(),
reservation: None,
budget,
}
}
/// Call only after the previous contents have been consumed. Growth charges
/// both allocations until the old buffer is freed, including after shrink.
pub(crate) fn ensure_len(&mut self, length: usize) -> io::Result<()> {
self.grow(length, 0)
}
/// Preserve an initialized prefix when replacing storage. Callers keep
/// `preserve` within both the existing allocation and the requested length.
pub(crate) fn grow(&mut self, length: usize, preserve: usize) -> io::Result<()> {
if self.bytes.len() >= length {
return Ok(());
}
let reservation = self.budget.as_ref().map(|budget| budget.reserve(length)).transpose()?;
let mut bytes = Vec::new();
bytes
.try_reserve_exact(length)
.map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
bytes.resize(length, 0);
bytes[..preserve].copy_from_slice(&self.bytes[..preserve]);
self.bytes = bytes.into_boxed_slice();
self.reservation = reservation;
Ok(())
}
/// Metadata is only a size hint: concurrent growth still reads through EOF.
/// Every replacement is charged before copying the retained prefix.
pub(crate) fn read_to_end(mut self, reader: &mut impl io::Read, hint: usize) -> io::Result<Bytes> {
self.ensure_len(hint)?;
let mut filled = 0;
loop {
if filled == self.bytes.len() {
// Probe EOF without forcing a larger allocation for an exact fit.
let mut probe = [0];
match reader.read(&mut probe) {
Ok(0) => break,
Ok(_) => {
let length = self.bytes.len().checked_mul(2).ok_or(io::ErrorKind::OutOfMemory)?.max(32);
self.grow(length, filled)?;
self.bytes[filled] = probe[0];
filled += 1;
}
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
Err(error) => return Err(error),
}
}
match reader.read(&mut self.bytes[filled..]) {
Ok(0) => break,
Ok(length) => filled += length,
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
Err(error) => return Err(error),
}
}
Ok(self.take_bytes(filled))
}
pub(crate) fn take_bytes(&mut self, length: usize) -> Bytes {
let bytes = std::mem::take(&mut self.bytes);
let reservation = self.reservation.take();
let mut bytes = Bytes::from(bytes.into_vec());
bytes.truncate(length);
Reservation::wrap(bytes, reservation)
}
pub(crate) fn memory_budget(&self) -> Option<Arc<BackgroundMemoryBudget>> {
self.budget.clone()
}
pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] {
&mut self.bytes
}
pub(crate) fn as_slice(&self) -> &[u8] {
&self.bytes
}
}
/// Bounded encoded-frame storage for serializer output and raw read staging.
/// Both paths reserve the actual allocation before initializing its bytes.
pub struct BackgroundEncodeBuffer {
buffer: BackgroundBuffer,
filled: usize,
maximum: usize,
}
impl BackgroundEncodeBuffer {
pub fn new(memory: Arc<BackgroundMemoryBudget>, maximum: usize) -> Self {
Self {
buffer: BackgroundBuffer::new(Some(memory)),
filled: 0,
maximum,
}
}
/// Borrow initialized raw-frame staging; decode only after a successful read.
/// Staging is not encoded output: `into_bytes` remains empty for this path.
pub fn prepare_read(&mut self, length: usize) -> io::Result<&mut [u8]> {
if length > self.maximum {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"background encoded frame exceeds its size limit",
));
}
self.buffer.ensure_len(length)?;
self.filled = 0;
Ok(&mut self.buffer.as_mut_slice()[..length])
}
pub fn into_bytes(mut self) -> Bytes {
self.buffer.take_bytes(self.filled)
}
}
impl io::Write for BackgroundEncodeBuffer {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
let required = self.filled.checked_add(bytes.len()).ok_or(io::ErrorKind::OutOfMemory)?;
if required > self.maximum {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"background encoded frame exceeds its size limit",
));
}
if required > self.buffer.as_slice().len() {
let capacity = self
.buffer
.as_slice()
.len()
.saturating_mul(2)
.max(32)
.max(required)
.min(self.maximum);
self.buffer.grow(capacity, self.filled)?;
}
self.buffer.as_mut_slice()[self.filled..required].copy_from_slice(bytes);
self.filled = required;
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn encoded_buffer_growth_retains_both_allocations_and_the_last_slice() {
use std::io::Write;
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(95, Duration::from_secs(30));
let mut buffer = BackgroundEncodeBuffer::new(owner.clone(), 64);
buffer.write_all(&[7; 32]).expect("initial allocation fits");
assert_eq!(owner.snapshot().used, 32);
let error = buffer.write_all(&[8]).expect_err("old 32 plus replacement 64 exceed 95");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert!(is_memory_pressure(&error));
assert_eq!(buffer.filled, 32);
assert_eq!(buffer.buffer.as_slice(), &[7; 32]);
assert_eq!(owner.snapshot().used, 32);
owner.set_sampled_limit(96, Duration::from_secs(30));
buffer.write_all(&[8]).expect("old 32 plus replacement 64 fit 96");
assert_eq!(owner.snapshot().used, 64);
let bytes = buffer.into_bytes();
assert_eq!(&bytes[..32], &[7; 32]);
assert_eq!(bytes[32], 8);
assert_eq!(bytes.len(), 33);
let slice = bytes.slice(32..);
drop(bytes);
assert_eq!(slice.as_ref(), &[8]);
assert_eq!(owner.snapshot().used, 64);
drop(slice);
assert_eq!(owner.snapshot().used, 0);
}
#[test]
fn encoded_buffer_checks_its_limit_before_reserving_storage() {
use std::io::Write;
let owner = BackgroundMemoryBudget::new();
for maximum in [0, 3] {
let mut buffer = BackgroundEncodeBuffer::new(owner.clone(), maximum);
buffer
.write_all(&[])
.expect("empty input needs no storage, including at limit zero");
let error = buffer
.write_all(&[7; 4])
.expect_err("oversized input must fail before memory admission");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert!(buffer.buffer.as_slice().is_empty());
assert!(buffer.into_bytes().is_empty());
assert_eq!(owner.snapshot().used, 0);
}
}
#[test]
fn returned_bytes_keep_the_allocation_charged_through_slices_and_shrink() {
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(8, Duration::from_secs(30));
let bytes = BackgroundBuffer::new(Some(owner.clone()))
.read_to_end(&mut &b"abcdefgh"[..], 8)
.expect("exact allocation fits without an EOF growth allocation");
assert_eq!(bytes.as_ref(), b"abcdefgh");
assert_eq!(owner.snapshot().used, 8);
let slice = bytes.slice(2..6);
owner.set_sampled_limit(0, Duration::ZERO);
drop(bytes);
assert_eq!(slice.as_ref(), b"cdef");
assert_eq!(owner.snapshot().used, 8, "the final slice still owns the complete allocation");
assert!(owner.reserve(1).is_err());
drop(slice);
assert_eq!(owner.snapshot().used, 0);
}
#[test]
fn whole_file_size_hint_never_truncates_growth_or_exposes_unused_bytes() {
for (hint, retained) in [(0, 32), (4, 32), (8, 8), (16, 16)] {
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(36, Duration::from_secs(30));
let bytes = BackgroundBuffer::new(Some(owner.clone()))
.read_to_end(&mut &b"abcdefgh"[..], hint)
.expect("read through EOF independently of the metadata hint");
assert_eq!(bytes.as_ref(), b"abcdefgh", "hint={hint}");
assert_eq!(owner.snapshot().used, retained, "charge allocated capacity, not visible length");
drop(bytes);
assert_eq!(owner.snapshot().used, 0);
}
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(16, Duration::from_secs(30));
let bytes = BackgroundBuffer::new(Some(owner.clone()))
.read_to_end(&mut std::io::empty(), 16)
.expect("file shrank to empty");
assert!(bytes.is_empty());
assert_eq!(owner.snapshot().used, 0, "empty results free unused storage before refund");
}
#[test]
fn preserving_growth_keeps_existing_payload_on_denial_and_reuse() {
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(11, Duration::from_secs(30));
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
buffer.ensure_len(4).expect("allocate initial payload");
buffer.as_mut_slice().copy_from_slice(b"data");
let error = buffer
.grow(8, 4)
.expect_err("old and replacement allocations need twelve bytes");
assert!(is_memory_pressure(&error));
assert_eq!(buffer.as_slice(), b"data");
assert_eq!(owner.snapshot().used, 4);
owner.set_sampled_limit(12, Duration::from_secs(30));
buffer.grow(8, 4).expect("preserve the payload after admission");
assert_eq!(buffer.as_slice(), b"data\0\0\0\0");
assert_eq!(owner.snapshot().used, 8);
owner.set_sampled_limit(0, Duration::ZERO);
buffer.grow(8, 4).expect("reuse retained storage after expiry");
assert_eq!(buffer.as_slice(), b"data\0\0\0\0");
let bytes = buffer.take_bytes(4);
drop(buffer);
assert_eq!(bytes.as_ref(), b"data");
assert_eq!(owner.snapshot().used, 8);
drop(bytes);
assert_eq!(owner.snapshot().used, 0);
}
#[test]
fn whole_file_growth_denial_and_source_failure_refund_storage() {
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(35, Duration::from_secs(30));
let error = BackgroundBuffer::new(Some(owner.clone()))
.read_to_end(&mut &b"abcdefgh"[..], 4)
.expect_err("old four plus new thirty-two bytes exceed capacity");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(owner.snapshot().used, 0);
struct FailedRead(u8);
impl io::Read for FailedRead {
fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
self.0 += 1;
match self.0 {
1 => Err(io::ErrorKind::Interrupted.into()),
2 => {
bytes[..2].copy_from_slice(b"ab");
Ok(2)
}
_ => Err(io::ErrorKind::PermissionDenied.into()),
}
}
}
let error = BackgroundBuffer::new(Some(owner.clone()))
.read_to_end(&mut FailedRead(0), 8)
.expect_err("failure after partial output is not successful EOF");
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
assert_eq!(owner.snapshot().used, 0);
}
#[tokio::test(start_paused = true)]
async fn shrink_and_expiry_keep_retained_storage_charged() {
let owner = BackgroundMemoryBudget::new();
assert!(owner.reserve(1).is_err());
owner.set_sampled_limit(16, Duration::from_secs(3));
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
buffer.ensure_len(8).expect("initial allocation");
let pointer = buffer.as_slice().as_ptr();
owner.set_sampled_limit(4, Duration::from_secs(3));
buffer.ensure_len(8).expect("existing storage can be reused after shrink");
assert_eq!(buffer.as_slice().as_ptr(), pointer);
assert_eq!(owner.snapshot(), BackgroundMemorySnapshot { limit: 4, used: 8 });
assert_eq!(buffer.ensure_len(9).expect_err("growth must be denied").kind(), io::ErrorKind::WouldBlock);
assert_eq!(owner.snapshot().used, 8, "failed growth preserves the existing reservation");
tokio::time::advance(Duration::from_secs(3)).await;
assert_eq!(owner.snapshot(), BackgroundMemorySnapshot { limit: 0, used: 8 });
assert!(owner.reserve(1).is_err(), "an expired sampler cannot grant memory");
drop(buffer);
assert_eq!(owner.snapshot().used, 0);
}
#[test]
fn growth_accounts_for_both_allocations_and_refunds_failed_attempts() {
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(12, Duration::from_secs(3));
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
buffer.ensure_len(4).expect("first allocation");
buffer.as_mut_slice().copy_from_slice(b"data");
assert!(buffer.ensure_len(9).is_err(), "four old plus nine new bytes exceed the cap");
assert_eq!(buffer.as_slice(), b"data");
buffer.ensure_len(8).expect("four old plus eight new bytes fit");
assert_eq!(owner.snapshot().used, 8);
assert_eq!(buffer.as_slice(), &[0; 8]);
drop(buffer);
assert_eq!(owner.snapshot().used, 0);
}
#[test]
fn concurrent_reservations_share_one_cap() {
let owner = BackgroundMemoryBudget::new();
owner.set_sampled_limit(1024, Duration::from_secs(30));
let ready = Arc::new(std::sync::Barrier::new(17));
let release = Arc::new(std::sync::Barrier::new(17));
// Start every worker before the first barrier; lazy spawning would deadlock.
let mut workers = Vec::with_capacity(16);
for _ in 0..16 {
let owner = owner.clone();
let ready = ready.clone();
let release = release.clone();
workers.push(std::thread::spawn(move || {
let reservation = owner.reserve(128).ok();
ready.wait();
release.wait();
reservation.is_some()
}));
}
ready.wait();
let retained = owner.snapshot().used;
release.wait();
let granted = workers
.into_iter()
.map(|worker| worker.join().expect("reservation worker"))
.filter(|granted| *granted)
.count();
assert_eq!(retained, 1024);
assert_eq!(granted, 8);
assert_eq!(owner.snapshot().used, 0);
}
}
@@ -1,443 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
const MAX_TOPOLOGY_NODES: usize = 256;
const MAX_TOPOLOGY_DEPTH: usize = 32;
const MAX_ATTRIBUTE_BYTES: u64 = 128;
/// Kernel device numbers can be reused after removal. `diskseq` distinguishes
/// those incarnations; this identity does not authorize storage mutations.
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub(super) struct PhysicalDeviceIdentity {
pub(super) major: u32,
pub(super) minor: u32,
pub(super) generation: u64,
}
fn invalid(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, message)
}
pub(super) fn attribute(path: &Path) -> io::Result<String> {
let mut value = String::new();
File::open(path)?.take(MAX_ATTRIBUTE_BYTES + 1).read_to_string(&mut value)?;
if value.len() > MAX_ATTRIBUTE_BYTES as usize {
return Err(invalid("physical device attribute exceeds its bound"));
}
Ok(value.trim().to_owned())
}
fn number(value: &str) -> io::Result<u64> {
if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
return Err(invalid("invalid physical device number"));
}
value.parse().map_err(|_| invalid("physical device number overflow"))
}
fn identity(path: &Path) -> io::Result<PhysicalDeviceIdentity> {
let generation = number(&attribute(&path.join("diskseq"))?)?;
if generation == 0 {
return Err(invalid("physical device generation is missing"));
}
let device = attribute(&path.join("dev"))?;
let (major, minor) = device
.split_once(':')
.ok_or_else(|| invalid("invalid physical device identity"))?;
let major = u32::try_from(number(major)?).map_err(|_| invalid("physical major number overflow"))?;
let minor = u32::try_from(number(minor)?).map_err(|_| invalid("physical minor number overflow"))?;
if number(&attribute(&path.join("diskseq"))?)? != generation {
return Err(invalid("physical device changed during discovery"));
}
Ok(PhysicalDeviceIdentity {
major,
minor,
generation,
})
}
#[derive(Debug, Eq, PartialEq)]
enum NodeTopology {
Partition {
device: String,
number: u64,
parent: PathBuf,
},
Whole {
identity: PhysicalDeviceIdentity,
children: Vec<PathBuf>,
},
}
fn canonical_beneath(path: &Path, root: &Path) -> io::Result<PathBuf> {
let path = path.canonicalize()?;
if !path.starts_with(root) {
return Err(invalid("physical device topology escapes sysfs"));
}
Ok(path)
}
fn node_topology(path: &Path, root: &Path) -> io::Result<NodeTopology> {
if canonical_beneath(path, root)? != path {
return Err(invalid("physical device path changed during discovery"));
}
match attribute(&path.join("partition")) {
Ok(partition) => {
let number = number(&partition)?;
if number == 0 {
return Err(invalid("physical partition number is missing"));
}
let parent = path.parent().ok_or_else(|| invalid("physical partition has no parent"))?;
return Ok(NodeTopology::Partition {
device: attribute(&path.join("dev"))?,
number,
parent: canonical_beneath(parent, root)?,
});
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
let identity = identity(path)?;
let mut children = Vec::new();
for entry in std::fs::read_dir(path.join("slaves"))? {
if children.len() >= MAX_TOPOLOGY_NODES {
return Err(invalid("physical device topology fanout exceeds its bound"));
}
children.push(canonical_beneath(&entry?.path(), root)?);
}
children.sort_unstable();
children.dedup();
Ok(NodeTopology::Whole { identity, children })
}
struct TopologySnapshot {
root: PathBuf,
entry: PathBuf,
first: PathBuf,
nodes: BTreeMap<PathBuf, NodeTopology>,
leaves: BTreeSet<PhysicalDeviceIdentity>,
}
impl TopologySnapshot {
fn discover(sysfs_root: &Path, major: u64, minor: u64) -> io::Result<Self> {
let root = sysfs_root.canonicalize()?;
let entry = root.join(format!("dev/block/{major}:{minor}"));
let first = canonical_beneath(&entry, &root)?;
if attribute(&first.join("dev"))? != format!("{major}:{minor}") {
return Err(invalid("physical device entry does not match its device number"));
}
let mut stack = vec![(first.clone(), 0, false)];
let mut active = BTreeSet::new();
let mut nodes = BTreeMap::new();
let mut leaves = BTreeMap::<PhysicalDeviceIdentity, PathBuf>::new();
while let Some((path, depth, exiting)) = stack.pop() {
if exiting {
active.remove(&path);
continue;
}
if depth > MAX_TOPOLOGY_DEPTH {
return Err(invalid("physical device topology is too deep"));
}
if active.contains(&path) {
return Err(invalid("physical device topology contains a cycle"));
}
if nodes.contains_key(&path) {
continue;
}
if nodes.len() >= MAX_TOPOLOGY_NODES {
return Err(invalid("physical device topology exceeds its node bound"));
}
let topology = node_topology(&path, &root)?;
active.insert(path.clone());
stack.push((path.clone(), depth, true));
match &topology {
NodeTopology::Partition { parent, .. } => stack.push((parent.clone(), depth + 1, false)),
NodeTopology::Whole { identity, children } => {
if stack.len() + children.len() > MAX_TOPOLOGY_NODES * 2 {
return Err(invalid("physical device topology fanout exceeds its bound"));
}
for child in children {
stack.push((child.clone(), depth + 1, false));
}
if children.is_empty()
&& let Some(previous) = leaves.insert(*identity, path.clone())
&& previous != path
{
return Err(invalid("physical device identity is ambiguous"));
}
}
}
nodes.insert(path, topology);
}
if leaves.is_empty() {
return Err(invalid("physical device topology has no leaves"));
}
Ok(Self {
root,
entry,
first,
nodes,
leaves: leaves.into_keys().collect(),
})
}
fn validate(&self) -> io::Result<()> {
if canonical_beneath(&self.entry, &self.root)? != self.first {
return Err(invalid("physical device entry changed during discovery"));
}
for (path, expected) in &self.nodes {
if &node_topology(path, &self.root)? != expected {
return Err(invalid("physical device topology changed during discovery"));
}
}
Ok(())
}
}
/// Resolve a bounded sysfs topology once, before installing an admission owner.
/// Missing topology or generation evidence is unknown, never a name-based owner.
/// Validate intermediate mappings too: stable leaves alone do not prove that a
/// device-mapper table still points to those leaves. This is an observation, not
/// a kernel topology lock; the admission owner must expire and refresh it.
pub(super) fn resolve_physical_devices(sysfs_root: &Path, major: u64, minor: u64) -> io::Result<Vec<PhysicalDeviceIdentity>> {
let snapshot = TopologySnapshot::discover(sysfs_root, major, minor)?;
snapshot.validate()?;
Ok(snapshot.leaves.into_iter().collect())
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
use std::os::unix::fs::symlink;
use std::sync::atomic::{AtomicU64, Ordering};
struct Fixture(PathBuf);
impl Fixture {
fn new() -> Self {
static NEXT: AtomicU64 = AtomicU64::new(0);
let stamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock")
.as_nanos();
let root = std::env::temp_dir().join(format!(
"physical-identity-{}-{stamp}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir(&root).expect("exclusive test root");
std::fs::create_dir_all(root.join("sys/dev/block")).expect("sysfs fixture");
Self(root)
}
fn root(&self) -> PathBuf {
self.0.join("sys")
}
fn device(&self, name: &str, major: u32, minor: u32, generation: u64) -> PathBuf {
let path = self.root().join("devices").join(name);
std::fs::create_dir_all(path.join("slaves")).expect("device fixture");
std::fs::write(path.join("dev"), format!("{major}:{minor}\n")).expect("device number");
std::fs::write(path.join("diskseq"), format!("{generation}\n")).expect("device generation");
symlink(&path, self.root().join(format!("dev/block/{major}:{minor}"))).expect("sysfs link");
path
}
fn link(&self, parent: &Path, child: &Path, name: &str) {
symlink(child, parent.join("slaves").join(name)).expect("slave link");
}
}
impl Drop for Fixture {
fn drop(&mut self) {
std::fs::remove_dir_all(&self.0).expect("remove owned fixture");
}
}
#[test]
fn partitions_share_the_leaf_generation_and_replacement_changes_identity() {
let fixture = Fixture::new();
let leaf = fixture.device("sda", 8, 0, 41);
let first = fixture.device("sda/sda1", 8, 1, 41);
let second = fixture.device("sda/sda2", 8, 2, 41);
std::fs::write(first.join("partition"), "1\n").expect("first partition");
std::fs::write(second.join("partition"), "2\n").expect("second partition");
let old = resolve_physical_devices(&fixture.root(), 8, 1).expect("first leaf");
assert_eq!(old, resolve_physical_devices(&fixture.root(), 8, 2).expect("alias leaf"));
assert_eq!(
old,
vec![PhysicalDeviceIdentity {
major: 8,
minor: 0,
generation: 41
}]
);
std::fs::write(leaf.join("diskseq"), "42\n").expect("replacement generation");
let new = resolve_physical_devices(&fixture.root(), 8, 1).expect("replacement leaf");
assert_ne!(old, new);
assert_eq!(new[0].generation, 42);
}
#[test]
fn overlapping_stacks_deduplicate_shared_physical_leaves() {
let fixture = Fixture::new();
let a = fixture.device("a", 8, 0, 1);
let b = fixture.device("b", 8, 16, 2);
let inner = fixture.device("dm-0", 253, 0, 3);
let outer = fixture.device("dm-1", 253, 1, 4);
fixture.link(&inner, &a, "a");
fixture.link(&inner, &b, "b");
fixture.link(&outer, &inner, "dm-0");
fixture.link(&outer, &a, "a-again");
assert_eq!(
resolve_physical_devices(&fixture.root(), 253, 0).expect("inner"),
resolve_physical_devices(&fixture.root(), 253, 1).expect("outer")
);
assert_eq!(resolve_physical_devices(&fixture.root(), 253, 1).expect("leaves").len(), 2);
}
#[test]
fn missing_or_invalid_generation_never_falls_back_to_a_device_name() {
let fixture = Fixture::new();
let leaf = fixture.device("a", 8, 0, 1);
for value in ["0", "-1", "18446744073709551616", "+1"] {
std::fs::write(leaf.join("diskseq"), value).expect("bad generation");
assert!(resolve_physical_devices(&fixture.root(), 8, 0).is_err(), "accepted {value}");
}
std::fs::remove_file(leaf.join("diskseq")).expect("missing generation");
assert_eq!(
resolve_physical_devices(&fixture.root(), 8, 0)
.expect_err("unknown generation")
.kind(),
io::ErrorKind::NotFound
);
assert!(resolve_physical_devices(&fixture.root(), 99, 99).is_err());
}
#[test]
fn topology_cycles_and_escaping_links_are_rejected() {
let fixture = Fixture::new();
let a = fixture.device("a", 8, 0, 1);
let b = fixture.device("b", 8, 16, 2);
fixture.link(&a, &b, "b");
fixture.link(&b, &a, "a");
assert!(
resolve_physical_devices(&fixture.root(), 8, 0)
.expect_err("cycle")
.to_string()
.contains("cycle")
);
std::fs::remove_file(b.join("slaves/a")).expect("remove cycle edge");
fixture.link(&b, &fixture.0, "outside");
assert!(
resolve_physical_devices(&fixture.root(), 8, 0)
.expect_err("escape")
.to_string()
.contains("escapes")
);
}
#[test]
fn intermediate_mapping_changes_are_rejected_even_when_old_leaves_survive() {
let fixture = Fixture::new();
let a = fixture.device("a", 8, 0, 1);
let b = fixture.device("b", 8, 16, 2);
let mapper = fixture.device("dm-0", 253, 0, 3);
fixture.link(&mapper, &a, "backing");
let snapshot = TopologySnapshot::discover(&fixture.root(), 253, 0).expect("initial mapping");
snapshot.validate().expect("unchanged topology");
std::fs::remove_file(mapper.join("slaves/backing")).expect("replace mapping");
fixture.link(&mapper, &b, "backing");
assert_eq!(identity(&a).expect("old leaf still exists").generation, 1);
assert!(snapshot.validate().is_err(), "stable old leaves cannot certify a changed mapper");
let current = resolve_physical_devices(&fixture.root(), 253, 0).expect("new complete mapping");
assert_eq!(current, vec![identity(&b).expect("new leaf")]);
let snapshot = TopologySnapshot::discover(&fixture.root(), 253, 0).expect("new mapping");
std::fs::write(mapper.join("diskseq"), "4\n").expect("replace mapper incarnation");
assert!(
snapshot.validate().is_err(),
"an intermediate generation change also invalidates discovery"
);
}
#[test]
fn entry_and_partition_changes_are_not_hidden_by_an_unchanged_parent() {
let fixture = Fixture::new();
fixture.device("a", 8, 0, 1);
let partition = fixture.device("a/p1", 8, 1, 1);
std::fs::write(partition.join("partition"), "1\n").expect("partition index");
let snapshot = TopologySnapshot::discover(&fixture.root(), 8, 1).expect("partition topology");
std::fs::write(partition.join("partition"), "2\n").expect("changed partition");
assert!(snapshot.validate().is_err());
let b = fixture.device("b", 8, 16, 2);
let snapshot = TopologySnapshot::discover(&fixture.root(), 8, 0).expect("first entry");
let entry = fixture.root().join("dev/block/8:0");
std::fs::remove_file(&entry).expect("replace entry");
symlink(&b, &entry).expect("changed entry target");
assert!(snapshot.validate().is_err());
assert!(
resolve_physical_devices(&fixture.root(), 8, 0).is_err(),
"entry device numbers must match"
);
}
#[test]
fn missing_slave_directory_is_unknown_not_a_physical_leaf() {
let fixture = Fixture::new();
let mapper = fixture.device("dm-0", 253, 0, 1);
std::fs::remove_dir(mapper.join("slaves")).expect("missing topology evidence");
assert!(resolve_physical_devices(&fixture.root(), 253, 0).is_err());
}
#[test]
fn topology_depth_and_attribute_sizes_are_bounded() {
let fixture = Fixture::new();
let first = fixture.device("d0", 253, 0, 1);
let mut previous = first.clone();
for index in 1..=MAX_TOPOLOGY_DEPTH + 1 {
let child = fixture.device(&format!("d{index}"), 253, u32::try_from(index).expect("bounded index"), 1);
fixture.link(&previous, &child, "next");
previous = child;
}
assert!(
resolve_physical_devices(&fixture.root(), 253, 0)
.expect_err("depth bound")
.to_string()
.contains("deep")
);
std::fs::remove_file(first.join("slaves/next")).expect("make first node a leaf");
std::fs::write(first.join("diskseq"), "1".repeat(129)).expect("oversized attribute");
assert!(
resolve_physical_devices(&fixture.root(), 253, 0)
.expect_err("attribute bound")
.to_string()
.contains("attribute")
);
}
#[test]
fn exact_node_bound_passes_and_one_more_node_is_rejected() {
let fixture = Fixture::new();
let root = fixture.device("root", 253, 0, 1);
for index in 0..MAX_TOPOLOGY_NODES - 1 {
let leaf = fixture.device(&format!("leaf-{index}"), 8, u32::try_from(index).expect("bounded index"), 2);
fixture.link(&root, &leaf, &format!("leaf-{index}"));
}
assert_eq!(
resolve_physical_devices(&fixture.root(), 253, 0)
.expect("exact node bound")
.len(),
MAX_TOPOLOGY_NODES - 1
);
let extra = fixture.device("extra", 9, 0, 3);
fixture.link(&root, &extra, "extra");
assert!(
resolve_physical_devices(&fixture.root(), 253, 0).is_err(),
"reject a graph above the node bound"
);
}
}
@@ -1,268 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::collections::{BTreeMap, BTreeSet};
use std::io;
use std::num::NonZeroU16;
use std::sync::{Arc, Weak};
use std::time::Duration;
use rustfs_concurrency::adaptive::AdaptiveAdmission;
#[cfg(test)]
use rustfs_concurrency::adaptive::{AdaptiveAdmissionSnapshot, AdaptivePermit};
use super::physical_identity::PhysicalDeviceIdentity;
const MAX_PHYSICAL_OWNERS: usize = 256;
/// Resolve aliases outside object work. The sampling owner serializes changes;
/// attached handles and native permits never consult this registry.
pub(super) struct PhysicalOwnerRegistry {
maximum: NonZeroU16,
owners: BTreeMap<PhysicalDeviceIdentity, Weak<AdaptiveAdmission>>,
}
/// One attachment to an existing physical capacity owner, not a reservation.
/// The gate itself owns permit slots, so native permit clones keep a registry
/// entry alive even after the last logical disk attachment disappears.
pub(super) struct PhysicalDeviceAdmission {
#[cfg(test)]
identity: PhysicalDeviceIdentity,
gate: Arc<AdaptiveAdmission>,
}
impl PhysicalOwnerRegistry {
pub(super) fn new(maximum: NonZeroU16) -> Self {
Self {
maximum,
owners: BTreeMap::new(),
}
}
/// Old topology descriptors may still reference an unused gate. Close it
/// before reclaiming its registry slot; only admitted I/O keeps an absent
/// incarnation alive, and its final release still reaches the same owner.
pub(super) fn retire_absent(&mut self, current: &BTreeSet<PhysicalDeviceIdentity>) {
self.owners.retain(|identity, owner| {
let Some(gate) = owner.upgrade() else {
return false;
};
if current.contains(identity) {
return true;
}
gate.set_sampled_limit(0, Duration::ZERO);
gate.snapshot().active != 0
});
}
pub(super) fn bind(&mut self, identities: &[PhysicalDeviceIdentity]) -> io::Result<Vec<PhysicalDeviceAdmission>> {
if identities.is_empty() || identities.len() > MAX_PHYSICAL_OWNERS || identities.iter().any(|id| id.generation == 0) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid physical capacity identities"));
}
let identities = identities.iter().copied().collect::<BTreeSet<_>>();
self.owners.retain(|_, owner| owner.strong_count() != 0);
let new_owners = identities
.iter()
.filter(|identity| !self.owners.contains_key(identity))
.count();
if new_owners > MAX_PHYSICAL_OWNERS - self.owners.len() {
return Err(io::Error::new(io::ErrorKind::WouldBlock, "physical capacity registry is full"));
}
Ok(identities
.into_iter()
.map(|identity| {
let gate = self.owners.get(&identity).and_then(Weak::upgrade).unwrap_or_else(|| {
let gate = AdaptiveAdmission::new(self.maximum, 0);
self.owners.insert(identity, Arc::downgrade(&gate));
gate
});
PhysicalDeviceAdmission {
#[cfg(test)]
identity,
gate,
}
})
.collect())
}
/// Only the sampling owner publishes capacity. New or expired identities
/// remain closed until a fresh observation establishes a bounded limit.
#[cfg(test)]
pub(super) fn publish(&mut self, identity: PhysicalDeviceIdentity, limit: u32, valid_for: Duration) -> bool {
let Some(gate) = self.owners.get(&identity).and_then(Weak::upgrade) else {
return false;
};
gate.set_sampled_limit(limit, valid_for);
true
}
}
impl PhysicalDeviceAdmission {
pub(super) fn gate(&self) -> &Arc<AdaptiveAdmission> {
&self.gate
}
#[cfg(test)]
pub(super) fn identity(&self) -> PhysicalDeviceIdentity {
self.identity
}
#[cfg(test)]
pub(super) fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
self.gate.snapshot()
}
#[cfg(test)]
pub(super) fn try_acquire(&self) -> Option<AdaptivePermit> {
self.gate.try_acquire()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn device(minor: u32, generation: u64) -> PhysicalDeviceIdentity {
PhysicalDeviceIdentity {
major: 259,
minor,
generation,
}
}
fn registry() -> PhysicalOwnerRegistry {
PhysicalOwnerRegistry::new(NonZeroU16::new(2).expect("two units"))
}
#[test]
fn aliases_and_overlapping_stacks_share_one_physical_capacity() {
let mut registry = registry();
let a = device(1, 11);
let b = device(2, 12);
let single = registry.bind(&[a]).expect("single device");
let stack = registry.bind(&[b, a, a]).expect("overlapping stack");
assert_eq!(stack.len(), 2, "duplicates cannot reserve additional units");
assert_eq!(stack[0].identity(), a);
assert!(single[0].try_acquire().is_none(), "unobserved capacity is closed");
assert!(registry.publish(a, 1, Duration::from_secs(60)));
let held = single[0].try_acquire().expect("physical unit");
assert!(stack[0].try_acquire().is_none(), "the alias cannot double the capacity");
assert_eq!(stack[0].snapshot().active, 1);
drop(held);
assert!(stack[0].try_acquire().is_some());
}
#[test]
fn native_tail_keeps_the_owner_alive_after_all_attachments_are_gone() {
let mut registry = registry();
let id = device(1, 11);
let attachment = registry.bind(&[id]).expect("logical disk");
assert!(registry.publish(id, 1, Duration::from_secs(60)));
let waiter = attachment[0].try_acquire().expect("admitted work");
let native = waiter.clone();
drop(waiter);
drop(attachment);
let reconnected = registry.bind(&[id]).expect("reconnect during native I/O");
assert_eq!(reconnected[0].snapshot().active, 1);
assert!(reconnected[0].try_acquire().is_none());
drop(native);
assert!(reconnected[0].try_acquire().is_some(), "only the final native release returns capacity");
}
#[test]
fn replacement_generation_does_not_receive_old_native_refunds() {
let mut registry = registry();
let old_id = device(1, 11);
let new_id = device(1, 12);
let old = registry.bind(&[old_id]).expect("old incarnation");
assert!(registry.publish(old_id, 1, Duration::from_secs(60)));
let native = old[0].try_acquire().expect("old native I/O");
drop(old);
let new = registry.bind(&[new_id]).expect("replacement incarnation");
assert_eq!(new[0].snapshot().active, 0);
assert!(new[0].try_acquire().is_none());
assert!(registry.publish(new_id, 1, Duration::from_secs(60)));
let current = new[0].try_acquire().expect("new work");
drop(native);
assert_eq!(new[0].snapshot().active, 1, "old completion cannot refund the replacement");
assert!(new[0].try_acquire().is_none());
drop(current);
assert_eq!(new[0].snapshot().active, 0);
}
#[test]
fn shrink_and_expiry_preserve_active_units_and_leave_other_disks_available() {
let mut registry = registry();
let a = device(1, 11);
let b = device(2, 12);
let disks = registry.bind(&[a, b]).expect("two physical disks");
assert!(registry.publish(a, 2, Duration::from_secs(60)));
assert!(registry.publish(b, 1, Duration::from_secs(60)));
let first = disks[0].try_acquire().expect("first unit");
let second = disks[0].try_acquire().expect("second unit");
assert!(registry.publish(a, 1, Duration::from_secs(60)));
assert_eq!(disks[0].snapshot().active, 2);
assert!(disks[0].try_acquire().is_none());
assert!(disks[1].try_acquire().is_some(), "a stalled disk cannot consume another disk's capacity");
assert!(registry.publish(a, 2, Duration::ZERO));
assert_eq!(disks[0].snapshot().limit, 0);
assert_eq!(disks[0].snapshot().active, 2);
drop((first, second));
assert!(disks[0].try_acquire().is_none(), "expired observations do not reopen after completion");
}
#[test]
fn registry_bound_preserves_live_native_owners_and_reclaims_only_dead_entries() {
let mut registry = registry();
let ids = (0..MAX_PHYSICAL_OWNERS)
.map(|minor| device(u32::try_from(minor).expect("bounded minor"), 1))
.collect::<Vec<_>>();
let attachments = registry.bind(&ids).expect("exact registry bound");
let extra = device(999, 1);
assert!(matches!(registry.bind(&[extra]), Err(error) if error.kind() == io::ErrorKind::WouldBlock));
assert_eq!(registry.owners.len(), MAX_PHYSICAL_OWNERS, "failed binding must not add partial owners");
assert!(registry.publish(ids[0], 1, Duration::from_secs(60)));
let native = attachments[0].try_acquire().expect("native tail");
drop(attachments);
let replacement = registry.bind(&[extra]).expect("dead owners reclaimed");
assert_eq!(registry.owners.len(), 2, "the native tail still owns its original entry");
let original = registry.bind(&[ids[0]]).expect("original native owner");
assert_eq!(original[0].snapshot().active, 1);
drop((native, original, replacement));
assert!(!registry.publish(extra, 1, Duration::from_secs(60)), "a dead owner cannot be refreshed");
assert!(registry.bind(&[]).is_err());
assert!(registry.bind(&[device(1, 0)]).is_err());
}
#[test]
fn retirement_reclaims_idle_descriptors_but_keeps_active_native_incarnations() {
let mut registry = registry();
let ids = (0..MAX_PHYSICAL_OWNERS)
.map(|minor| device(u32::try_from(minor).expect("bounded minor"), 1))
.collect::<Vec<_>>();
let descriptors = registry.bind(&ids).expect("full registry");
assert!(registry.publish(ids[0], 1, Duration::from_secs(60)));
let native = descriptors[0].try_acquire().expect("active old incarnation");
let mut current = ids[2..].iter().copied().collect::<BTreeSet<_>>();
registry.retire_absent(&current);
assert_eq!(registry.owners.len(), MAX_PHYSICAL_OWNERS - 1);
assert_eq!(descriptors[0].snapshot().active, 1);
assert!(descriptors[0].try_acquire().is_none());
assert!(descriptors[1].try_acquire().is_none());
let first = device(0, 2);
let second = device(1, 2);
let replacement = registry.bind(&[first]).expect("idle descriptor was reclaimed");
current.insert(first);
assert!(matches!(registry.bind(&[second]), Err(error) if error.kind() == io::ErrorKind::WouldBlock));
drop(native);
registry.retire_absent(&current);
let next = registry.bind(&[second]).expect("final native release permits reclamation");
assert_eq!(replacement[0].snapshot().active, 0);
assert_eq!(next[0].snapshot().active, 0);
assert_eq!(descriptors[0].snapshot().active, 0);
assert!(
!registry.publish(ids[0], 1, Duration::from_secs(60)),
"retired identity cannot be refreshed"
);
}
}
File diff suppressed because it is too large Load Diff
@@ -1,309 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::{PhysicalDiskBinding, ResourceIoUnit};
use rustfs_concurrency::adaptive::{
NodeAdmission,
permit_set::{NodeResourceCompletion, NodeResourcePermit},
};
use std::cell::{Cell, RefCell};
use std::future::Future;
use std::io;
use std::sync::Arc;
tokio::task_local! {
static REMOTE_RESOURCES: RemoteBackgroundIo;
}
/// Receiver-owned admission for an authenticated background RPC. The owner must
/// be the same node controller used by local Scanner and Heal work.
pub struct RemoteBackgroundIo {
node: Arc<NodeAdmission>,
admitted: RefCell<Option<NodeResourcePermit>>,
progressed: Cell<bool>,
finished: Cell<bool>,
}
impl RemoteBackgroundIo {
pub fn new(node: Arc<NodeAdmission>) -> Self {
Self {
node,
admitted: RefCell::new(None),
progressed: Cell::new(false),
finished: Cell::new(false),
}
}
/// Move the request's reservations into its existing stream producer.
/// Only physical subsets may be cloned into native I/O; never its node unit.
pub fn take_current() -> Option<Self> {
REMOTE_RESOURCES
.try_with(|context| Self {
node: context.node.clone(),
admitted: RefCell::new(context.admitted.take()),
progressed: Cell::new(context.progressed.replace(false)),
finished: Cell::new(false),
})
.ok()
}
/// Defer a response's progress without keeping async admission alive while
/// its body streams. The reader retains its own native resource ownership.
pub fn take_current_completion() -> Option<NodeResourceCompletion> {
REMOTE_RESOURCES
.try_with(|context| {
let admitted = context.admitted.take();
if context.progressed.replace(false) {
admitted.map(NodeResourcePermit::into_completion)
} else {
None
}
})
.ok()
.flatten()
}
/// A failed response or cancelled producer must not credit earlier phases.
pub fn discard_current_progress() {
let _ = REMOTE_RESOURCES.try_with(|context| context.progressed.set(false));
}
}
impl Drop for RemoteBackgroundIo {
fn drop(&mut self) {
if self.finished.get()
&& self.progressed.get()
&& let Some(permit) = self.admitted.get_mut().take()
{
permit.complete();
}
}
}
/// Scope a verified request or its stream producer. An interrupted request
/// returns node capacity while native jobs retain their selected physical units.
pub async fn with_remote_background_io<F: Future>(context: Option<RemoteBackgroundIo>, operation: F) -> F::Output {
match context {
Some(context) => {
REMOTE_RESOURCES
.scope(context, async {
let result = operation.await;
REMOTE_RESOURCES.with(|context| context.finished.set(true));
result
})
.await
}
None => operation.await,
}
}
pub(crate) fn active() -> bool {
REMOTE_RESOURCES.try_with(|_| ()).is_ok()
}
pub(crate) fn resources() -> Option<ResourceIoUnit> {
REMOTE_RESOURCES
.try_with(|context| {
context
.admitted
.borrow()
.as_ref()
.map(|permit| ResourceIoUnit::from(permit.resources().clone()))
})
.ok()
.flatten()
}
pub(crate) fn record_progress() {
let _ = REMOTE_RESOURCES.try_with(|context| context.progressed.set(true));
}
pub(crate) fn admit(binding: &PhysicalDiskBinding) -> io::Result<()> {
REMOTE_RESOURCES
.try_with(|context| {
let deferred = || io::Error::from(io::ErrorKind::WouldBlock);
let plan = binding.plan().ok_or_else(deferred)?;
let mut admitted = context.admitted.borrow_mut();
if let Some(permit) = admitted.as_ref() {
// Reuse the request's receipt for append/read/rename phases at
// capacity one. Additional attachments never wait under locks.
let native = plan.capture_or_acquire_native(permit.resources()).map_err(|_| deferred())?;
drop(native);
if !plan.is_current() {
return Err(deferred());
}
} else {
*admitted = Some(plan.try_acquire(&context.node).ok_or_else(deferred)?);
}
Ok(())
})
.unwrap_or(Ok(()))
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_concurrency::adaptive::{
AdaptiveAdmission, FeedbackController, FeedbackObservation, ResourcePressure, permit_set::AdaptivePermitSetOwner,
};
use std::num::NonZeroU16;
fn node() -> Arc<NodeAdmission> {
let node = NodeAdmission::new(NonZeroU16::MIN);
node.update(
&mut FeedbackController::new(),
Some(FeedbackObservation {
generation: 1,
max_concurrency: 1,
pressure: ResourcePressure::Unknown,
completed_units: None,
}),
);
node
}
#[tokio::test]
async fn receiver_reuses_one_receipt_across_phases_and_releases_only_native_subsets() {
let node = node();
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("device catalog");
let first = PhysicalDiskBinding::default();
let second = PhysicalDiskBinding::default();
first
.set_test_shared_capacity(shared.clone(), vec![a.clone()])
.expect("device A");
second.set_test_shared_capacity(shared, vec![b.clone()]).expect("device B");
let native = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
assert!(resources().is_none(), "no physical work before target selection");
admit(&first).expect("first phase");
let admitted = resources().expect("admitted resources");
let native = first
.plan()
.expect("current plan")
.capture_native(admitted.as_set().expect("resource set"))
.expect("escaped native read");
admit(&first).expect("second phase reuses capacity while the first reader lives");
admit(&second).expect("additional selected device");
assert_eq!([node.snapshot().active, a.snapshot().active, b.snapshot().active], [1, 1, 1]);
record_progress();
native
})
.await;
assert_eq!([node.snapshot().active, a.snapshot().active, b.snapshot().active], [0, 1, 0]);
assert_eq!([node.completed(), a.completed(), b.completed()], [Some(1), Some(1), Some(1)]);
drop(native);
assert_eq!(a.snapshot().active, 0);
}
#[tokio::test]
async fn receiver_cancellation_returns_node_but_not_escaped_native_capacity() {
let node = node();
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let binding = PhysicalDiskBinding::default();
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
let (sender, receiver) = tokio::sync::oneshot::channel();
let task = tokio::spawn(with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async move {
admit(&binding).expect("receiver admission");
let admitted = resources().expect("receiver resources");
let native = binding
.plan()
.expect("plan")
.capture_native(admitted.as_set().expect("set"))
.expect("native subset");
sender.send(native).expect("native owner receiver");
record_progress();
std::future::pending::<()>().await;
}));
let native = receiver.await.expect("native read ownership");
task.abort();
assert!(task.await.expect_err("cancel request").is_cancelled());
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 1]);
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
drop(native);
assert_eq!(gate.snapshot().active, 0);
}
#[tokio::test]
async fn receiver_defers_stream_progress_without_retaining_node_capacity() {
for (progressed, succeeds) in [(false, true), (true, false), (true, true)] {
let node = node();
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let binding = PhysicalDiskBinding::default();
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
let completion = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
admit(&binding).expect("receiver admission");
if progressed {
record_progress();
}
let completion = RemoteBackgroundIo::take_current_completion();
assert!(RemoteBackgroundIo::take_current_completion().is_none(), "single progress owner");
assert!(resources().is_none());
completion
})
.await;
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
assert_eq!(completion.is_some(), progressed);
if succeeds && let Some(completion) = completion {
completion.complete();
}
let expected = u64::from(progressed && succeeds);
assert_eq!([node.completed(), gate.completed()], [Some(expected), Some(expected)]);
}
assert!(
RemoteBackgroundIo::take_current_completion().is_none(),
"foreground has no progress owner"
);
}
#[tokio::test]
async fn receiver_failed_completion_does_not_credit_successful_preparation() {
let node = node();
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let binding = PhysicalDiskBinding::default();
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
admit(&binding).expect("receiver admission");
record_progress();
RemoteBackgroundIo::discard_current_progress();
})
.await;
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
}
#[tokio::test]
async fn receiver_stream_transfer_moves_node_ownership_and_defers_unavailable_devices() {
let node = node();
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
let binding = PhysicalDiskBinding::default();
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
let context = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
assert_eq!(admit(&binding).expect_err("closed device").kind(), io::ErrorKind::WouldBlock);
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
gate.set_limit(1);
admit(&binding).expect("opened device");
let context = RemoteBackgroundIo::take_current().expect("stream context");
assert!(resources().is_none(), "request cannot keep a second node owner");
context
})
.await;
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
assert_eq!(node.completed(), Some(0));
with_remote_background_io(Some(context), async {
admit(&binding).expect("stream reuses the request's receipt");
binding
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
.expect("replace physical device");
assert_eq!(
admit(&binding).expect_err("old receipt cannot cross topology").kind(),
io::ErrorKind::WouldBlock
);
})
.await;
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
assert_eq!(node.completed(), Some(0), "admission alone is not completed work");
}
}
File diff suppressed because it is too large Load Diff
+68 -339
View File
@@ -28,7 +28,7 @@ use super::{
should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
};
#[cfg(test)]
use super::{run_inline_before_file_sync_admission, run_owned_file_before_open, run_rename_data_after_first_publication};
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
use crate::crash_inject::{self, CrashPoint};
use crate::disk::{
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken,
@@ -175,28 +175,17 @@ pub(super) fn create_local_inline_rollback_backup(
dst_file_path: &Path,
staging_file_path: &Path,
old_metadata: &[u8],
sync: bool,
) -> std::io::Result<PathBuf> {
let Some(staging_parent) = staging_file_path.parent() else {
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
};
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
remove_file_if_exists(&backup_path)?;
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
let write_res = (|| -> std::io::Result<()> {
let mut file = std::fs::File::create(&backup_path)?;
std::io::Write::write_all(&mut file, old_metadata)?;
if sync && !old_metadata.is_empty() {
file.sync_data()?;
}
Ok(())
})();
if let Err(err) = write_res {
let _ = remove_file_if_exists(&backup_path);
return Err(err);
}
} else if sync && !old_metadata.is_empty() {
std::fs::File::open(&backup_path)?.sync_data()?;
if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
&& let Err(err) = std::fs::write(&backup_path, old_metadata)
{
let _ = remove_file_if_exists(&backup_path);
return Err(err);
}
Ok(backup_path)
}
@@ -340,38 +329,6 @@ pub(super) async fn lock_rename_commit_directories(
result.map_err(to_file_error).map_err(DiskError::from)
}
#[cfg(target_os = "linux")]
fn prepare_rename_commit_and_destination_metadata_std(
create_source_parent: bool,
source_parent: &Path,
destination_parent: &Path,
destination_metadata: &Path,
base_dir: &Path,
publication_root: &os::PublicationRoot,
) -> std::io::Result<(os::RenameCommitGuard, Option<Bytes>)> {
if create_source_parent {
std::fs::create_dir_all(source_parent)?;
}
let guard = match os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) {
Ok(guard) => guard,
Err(err) => {
return Err(match std::fs::symlink_metadata(base_dir) {
Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
_ => err,
});
}
};
let data = os::read_destination_file_with_commit_guard(destination_metadata, &guard)?;
Ok((guard, data.map(Bytes::from)))
}
#[cfg(target_os = "linux")]
enum FusedCommitAttempt {
Committed(RenameDataResp),
CommitErr(DiskError),
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
}
async fn prepare_rename_commit_and_destination_metadata(
create_source_parent: bool,
source_parent: &Path,
@@ -614,9 +571,6 @@ impl LocalDisk {
return Err(err);
}
}
if dst_volume == crate::disk::RUSTFS_META_BUCKET {
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
}
if fi.is_legacy_indexed_delete_marker() {
fi.erasure.index = 0;
}
@@ -744,242 +698,6 @@ impl LocalDisk {
let dst_file_parent = dst_file_path
.parent()
.ok_or_else(|| DiskError::other("missing object metadata parent"))?;
#[cfg(target_os = "linux")]
let (rename_commit_guard, has_dst_buf) = if !preserve_upload_metadata
&& multipart_publication.is_none()
&& pool_meta_condition.is_none()
&& !fi_healing
&& !os::dst_dir_fsync_group_commit_enabled()
&& !os::file_fdatasync_group_commit_enabled()
&& !super::has_rename_data_test_failpoints(dst_path)
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
&& !super::has_delete_version_test_failpoints()
{
#[cfg(test)]
if !no_inline {
run_inline_before_file_sync_admission(dst_path);
}
let sync_data_shards = no_inline && durability.syncs_data_shards();
let sync_commit_metadata = durability.syncs_commit_metadata();
let file_sync_admission = if sync_data_shards || sync_commit_metadata {
Some(
os::acquire_file_sync_admission(self.file_sync_permits.clone())
.await
.map_err(to_file_error)
.map_err(DiskError::from)?,
)
} else {
None
};
let src_parent_owned = src_file_parent.to_path_buf();
let dst_parent_owned = dst_file_parent.to_path_buf();
let src_file_path_owned = src_file_path.clone();
let dst_file_path_owned = dst_file_path.clone();
let dst_volume_dir_owned = dst_volume_dir.clone();
let publication_root_owned = self.publication_root.clone();
let has_data_dir_path_owned = has_data_dir_path.clone();
let dst_volume_str = dst_volume.to_string();
let dst_path_str = dst_path.to_string();
let fi_moved = fi;
#[cfg(test)]
let root_owned = self.root.clone();
let fused_attempt = move |is_cancelled: &dyn Fn() -> bool| -> std::io::Result<FusedCommitAttempt> {
let (guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata_std(
!no_inline,
&src_parent_owned,
&dst_parent_owned,
&dst_file_path_owned,
&dst_volume_dir_owned,
&publication_root_owned,
)?;
// An existing xl.meta that replaces an existing version slot
// requires a rollback directory; appending a new version keeps
// an inline rollback backup for post-rename directory sync errors.
if is_cancelled() {
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
}
let mut xlmeta = FileMeta::new();
let mut dst_meta_unparsable = false;
if let Some(ref dst_buf) = has_dst_buf {
if FileMeta::is_xl2_v1_format(dst_buf)
&& let Ok(nmeta) = FileMeta::load(dst_buf)
{
xlmeta = nmeta;
} else {
dst_meta_unparsable = true;
}
}
let version_id = fi_moved.version_id.unwrap_or_default();
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
let rollback_data_dir = has_old_data_dir.or_else(|| {
if has_dst_buf.is_some() && xlmeta.find_version(Some(version_id)).is_ok() {
Some(inline_metadata_rollback_dir(version_id, &xlmeta))
} else {
None
}
});
if rollback_data_dir.is_some() {
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
}
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
has_dst_buf.is_some(),
dst_meta_unparsable,
&xlmeta,
&dst_volume_str,
&dst_path_str,
version_id,
);
xlmeta.add_version(fi_moved)?;
let version_signature = rename_data_versions_signature(&xlmeta);
let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
let destination_was_absent = has_dst_buf.is_none();
let mut skip_parent = dst_volume_dir_owned.clone();
if has_dst_buf.is_some() {
skip_parent = dst_parent_owned.clone();
}
let mut prepared_metadata_source =
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
let mut staged_rollback_path = None;
if sync_commit_metadata && let Some(ref old_metadata) = has_dst_buf {
let backup_path =
create_local_inline_rollback_backup(&dst_file_path_owned, &src_file_path_owned, old_metadata, true)?;
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
}
if no_inline {
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
// The payload was synced, but its part.1 directory entry
// must also be persisted before moving the data directory.
os::fsync_dir_std(src_data_path)
} else {
os::sync_dir_files_std(src_data_path)
};
if let Err(err) = sync_result
&& err.kind() != ErrorKind::NotFound
{
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
return Err(err);
}
}
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
{
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
return Ok(FusedCommitAttempt::CommitErr(err));
}
#[cfg(test)]
if has_data_dir_path_owned.is_some() {
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
}
}
if let Err(err) = os::rename_prepared_source_on_thread(
prepared_metadata_source,
&src_file_path_owned,
&dst_file_path_owned,
&skip_parent,
&publication_root_owned,
&guard,
) {
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
let _ = std::fs::rename(dst_data_path, src_data_path);
}
return Ok(FusedCommitAttempt::CommitErr(err));
}
if sync_commit_metadata
&& let Err(err) = fsync_inline_commit_std(
&dst_file_path_owned,
&dst_volume_dir_owned,
destination_was_absent,
staged_rollback_path.as_ref(),
&guard,
)
{
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
let _ = std::fs::rename(dst_data_path, src_data_path);
}
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
}
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
if let Some(parent) = src_file_path_owned.parent() {
let _ = std::fs::remove_dir(parent);
}
Ok(FusedCommitAttempt::Committed(RenameDataResp {
multipart_source: None,
old_data_dir: None,
rollback_data_dir: None,
cleanup_data_dir: None,
sign: version_signature,
old_current_size,
old_current_source_checked,
old_current_source,
}))
};
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
os::run_cancellable_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, fused_attempt).await
} else {
os::run_cancellable_blocking_namespace_operation(mutation_lease.clone(), fused_attempt).await
}
.map_err(to_file_error)
.map_err(DiskError::from);
drop(file_sync_admission);
match attempt_res {
Ok(FusedCommitAttempt::Committed(resp)) => {
for part_path in &invalidate_part_paths {
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
}
#[cfg(unix)]
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
receipt_mutation.complete();
}
return Ok(resp);
}
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
for part_path in &invalidate_part_paths {
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
}
return Err(err);
}
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
fi = restored_fi;
(guard, dst_buf)
}
}
} else {
prepare_rename_commit_and_destination_metadata(
!no_inline && !preserve_upload_metadata,
src_file_parent,
dst_file_parent,
&dst_file_path,
&dst_volume_dir,
&self.publication_root,
mutation_lease.clone(),
)
.await?
};
#[cfg(not(target_os = "linux"))]
// Acquire the common trees before reading destination metadata. On
// Windows this pins the object directory identity across publication.
// Multipart authority already created its source tree. Recreating it
@@ -1090,6 +808,52 @@ impl LocalDisk {
// after the commit rename, and a crash before the rename means the
// PUT was never acknowledged. A metadata commit: relaxed tiers
// leave it to the page cache.
if let Some(publication) = multipart_publication.as_ref() {
pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
}
let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
SyncMode::FileOnly
} else {
SyncMode::None
};
// The tmp xl.meta write and the shard-file fdatasync are independent
// (disjoint paths) and both only need to be durable before the commit
// renames below, so run them concurrently to drop a blocking
// round-trip from the PUT commit critical path (rustfs/backlog#922
// step 2). The "contents durable -> rename -> dst dir fsync" ordering
// is unchanged — both futures complete before any rename — which the
// rename_data crash-consistency harness (backlog#935) exercises.
//
// Shard durability: once rename_data succeeds the write is
// acknowledged, so data must not live only in the page cache.
// Multipart parts were already synced during rename_part, so their
// fdatasync here is a cheap no-op. A missing source dir is left for the
// rename below to report through the existing rollback path. Payload
// durability is kept by both strict and relaxed.
let tmp_meta_write = {
let src_file_path = multipart_publication
.as_ref()
.map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
let dst_file_path = dst_file_path.clone();
let owned_directory_guard = commit.directory_guard.clone();
let owned_mutation_lease = commit.mutation_lease.clone();
async move {
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
#[cfg(test)]
run_owned_file_write_before_open(&src_file_path);
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
&src_file_path,
&dst_file_path,
&owned_directory_guard,
)?;
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
Ok(prepared_metadata_source)
})
.await
.map_err(to_file_error)
.map_err(DiskError::from)
}
};
let shard_sync = async {
if durability.syncs_data_shards()
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
@@ -1100,55 +864,15 @@ impl LocalDisk {
}
Ok(())
};
let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
let dst_file_path_ref = &dst_file_path;
let dir_guard_ref = &commit.directory_guard;
let lease_clone = commit.mutation_lease.clone();
let prepare_fut = pin_multipart_future(|| {
publication.prepare_and_install_head(
&new_dst_buf,
has_dst_buf.as_deref(),
dst_file_path_ref,
dir_guard_ref,
lease_clone,
)
});
let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
let prepared_metadata_source = tmp_meta_res?;
shard_sync_res?;
prepared_metadata_source
} else {
let tmp_meta_sync = if durability.syncs_commit_metadata() {
SyncMode::FileOnly
} else {
SyncMode::None
};
let tmp_meta_write = {
let src_file_path = commit.source_metadata.to_path_buf();
let dst_file_path = dst_file_path.clone();
let owned_directory_guard = commit.directory_guard.clone();
let owned_mutation_lease = commit.mutation_lease.clone();
async move {
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
#[cfg(test)]
run_owned_file_before_open(&src_file_path);
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
&src_file_path,
&dst_file_path,
&owned_directory_guard,
)?;
prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
Ok(prepared_metadata_source)
})
.await
.map_err(to_file_error)
.map_err(DiskError::from)
}
};
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
let prepared_metadata_source = tmp_meta_res?;
shard_sync_res?;
prepared_metadata_source
let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
// Surface a tmp-meta failure first (its prior serial position), then a
// shard-sync failure; either aborts before any rename, exactly as the
// sequential version did.
let prepared_metadata_source = tmp_meta_res?;
shard_sync_res?;
let prepared_metadata_source = match multipart_publication.as_ref() {
Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
None => prepared_metadata_source,
};
commit.directory_guard = remove_dst_base_before_commit(
dst_path,
@@ -1302,7 +1026,7 @@ impl LocalDisk {
// deterministic rollback backup.
let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
#[cfg(test)]
run_owned_file_before_open(&backup_path);
run_owned_file_write_before_open(&backup_path);
backup_path_guard.write_file_for_path_access(
&backup_path,
backup_bytes.as_ref(),
@@ -1694,8 +1418,13 @@ impl LocalDisk {
#[cfg(not(windows))]
let backup_path = {
let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
create_local_inline_rollback_backup(&dst, &src, old_metadata)?
};
#[cfg(not(windows))]
if sync {
let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
std::fs::File::open(&backup_path)?.sync_data()?;
}
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
}
@@ -1317,7 +1317,7 @@ impl LocalDisk {
continue;
};
#[cfg(test)]
run_owned_file_before_open(&record_destination.join(name));
run_owned_file_write_before_open(&record_destination.join(name));
let mut file = std::fs::File::create(staged.path().join(name))?;
std::io::Write::write_all(&mut file, bytes)?;
if sync && !bytes.is_empty() {
@@ -1795,7 +1795,7 @@ impl LocalDisk {
self.write_all_meta_with_namespace_owner(
volume,
&format!("{object}/{STORAGE_FORMAT_FILE}"),
target,
&target,
true,
owner.clone(),
)
@@ -2090,550 +2090,6 @@ impl LocalDisk {
pub(super) async fn delete_recovery_barrier(&self, volume: &str, object: &str, transaction: Uuid, stage: &str) -> Result<()> {
barrier(volume, object, transaction, stage, &self.root).await
}
pub(super) async fn try_fused_recoverable_delete_version(
&self,
volume: &str,
object: &str,
fi: &FileInfo,
opts: &DeleteOptions,
transaction: Uuid,
owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<()>> {
if !opts.recoverable_delete
|| opts.undo_write
|| opts.expected_delete_marker.is_some()
|| pool_meta_native::is_object(&self.root, volume, object)
|| has_delete_version_test_failpoints()
{
return Ok(None);
}
self.reject_multipart_delete_recovery_mutation(volume, object)?;
self.wait_for_startup_cleanup().await;
let (_, file_path) = self.io_get_bucket_and_object_path(volume, object)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
let destination = file_path.join(transaction.to_string());
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let sync = effective_durability(volume).syncs_commit_metadata();
let incarnation = opts.delete_recovery_bucket_incarnation;
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
let fi_clone = fi.clone();
let volume_str = volume.to_owned();
let object_str = object.to_owned();
let res = os::run_blocking_namespace_operation_with_sync(
lease,
sync,
move || -> std::io::Result<Result<Option<Option<Uuid>>>> {
let buf = match std::fs::read(&xl_path) {
Ok(buf) => buf,
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
Err(err) => return Err(err),
};
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>);
let (old_dir, target, journal_raw) = match (|| -> Result<Option<PreparedDeleteTuple>> {
let mut meta = FileMeta::load(&buf)?;
let mut deleted_headers: Vec<_> = meta
.versions
.iter()
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
.collect();
let original_versions: Vec<_> = meta
.versions
.iter()
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
.collect::<Result<_>>()?;
let old_dir = meta.delete_version(&fi_clone)?;
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
return Err(DiskError::FileCorrupt);
}
if let Some(uuid) = old_dir {
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
check_path_length(old_path.to_string_lossy().as_ref())?;
if old_path.join(STORAGE_FORMAT_FILE).exists() {
return Ok(None);
}
}
let surviving: HashSet<_> = meta
.versions
.iter()
.map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
.collect();
deleted_headers.retain(|header| !surviving.contains(header));
let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
let mut predecessors = BTreeSet::new();
if let Ok(entries) = std::fs::read_dir(&file_path) {
for (visited, entry) in entries.enumerate() {
if visited >= 65536 {
return Err(DiskError::DiskOngoingReq);
}
let entry = entry.map_err(to_file_error)?;
let Some(id) = entry
.file_name()
.to_str()
.and_then(|name| Uuid::parse_str(name).ok())
.filter(|id| !id.is_nil() && *id != transaction)
else {
continue;
};
if !entry.file_type().map_err(to_file_error)?.is_dir() {
return Err(DiskError::InvalidPath);
}
let prepared = regular_bytes(&entry.path().join(PREPARED))?;
let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
let mut records = Vec::new();
if let Some(bytes) = prepared {
records.push(
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
.map_err(|_| DiskError::FileCorrupt)?,
);
}
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
records.push(
serde_json::from_slice::<CommitReceipt>(&bytes)
.map_err(|_| DiskError::FileCorrupt)?
.journal,
);
}
for record in records {
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|| record.volume != volume_str
|| record.object != object_str
|| record.transaction != id
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
{
return Err(DiskError::FileCorrupt);
}
predecessors.insert(id);
}
}
}
let mut sparse = fi_clone;
sparse.data = None;
let journal = DeleteRecoveryJournal {
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
volume: volume_str,
object: object_str,
transaction,
bucket_incarnation: incarnation,
original_digest: Some(digest(&buf)),
target_digest: target.as_deref().map(digest),
data_dirs: old_dir.into_iter().collect(),
deleted_versions,
requests: vec![sparse],
original_versions,
predecessors: predecessors.into_iter().collect(),
};
Ok(Some((old_dir, target, journal_bytes(&journal)?)))
})() {
Ok(Some(tuple)) => tuple,
Ok(None) => return Ok(Ok(None)),
Err(err) => return Ok(Err(err)),
};
std::fs::create_dir_all(&staging_root)?;
let staged = tempfile::Builder::new()
.prefix("delete-preparation-")
.tempdir_in(&staging_root)?;
let backup_path = staged.path().join(STORAGE_FORMAT_FILE_BACKUP);
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
let mut file = std::fs::File::create(&backup_path)?;
std::io::Write::write_all(&mut file, &buf)?;
if sync && !buf.is_empty() {
file.sync_data()?;
}
}
for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
let Some(bytes) = bytes else {
continue;
};
let mut file = std::fs::File::create(staged.path().join(name))?;
std::io::Write::write_all(&mut file, bytes)?;
if sync && !bytes.is_empty() {
file.sync_data()?;
}
}
if sync {
os::fsync_dir_std(staged.path())?;
}
match std::fs::symlink_metadata(&destination) {
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
Err(err) => return Err(err),
},
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
if let Some(parent) = destination.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::rename(staged.path(), &destination)?;
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
let mut reserved_data_path = None;
if let Some(uuid) = old_dir {
let old_path = file_path.join(uuid.to_string());
if old_path.is_dir() {
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
let reserve_res = (|| -> std::io::Result<()> {
let marker = std::fs::File::create(&reserved_marker)?;
if sync {
marker.sync_all()?;
os::fsync_dir_std(&old_path)?;
}
Ok(())
})();
if let Err(err) = reserve_res {
let _ = std::fs::remove_file(&reserved_marker);
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
reserved_data_path = Some((old_path, reserved_marker));
}
}
if let Some(ref target_bytes) = target {
let mut staged_xl = match tempfile::NamedTempFile::new_in(&staging_root) {
Ok(staged_xl) => staged_xl,
Err(err) => {
if let Some((_, reserved_marker)) = &reserved_data_path {
let _ = std::fs::remove_file(reserved_marker);
}
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
};
let write_res = (|| -> std::io::Result<()> {
std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
if sync && !target_bytes.is_empty() {
staged_xl.as_file().sync_data()?;
}
std::fs::rename(staged_xl.path(), &xl_path)
})();
if let Err(err) = write_res {
if let Some((_, reserved_marker)) = &reserved_data_path {
let _ = std::fs::remove_file(reserved_marker);
}
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
} else if let Err(err) = std::fs::remove_file(&xl_path)
&& err.kind() != ErrorKind::NotFound
{
if let Some((_, reserved_marker)) = &reserved_data_path {
let _ = std::fs::remove_file(reserved_marker);
}
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
if sync {
os::fsync_dir_std(&file_path)?;
}
if let Some((old_path, reserved_marker)) = reserved_data_path {
let recovery_marker = old_path.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
match std::fs::rename(&reserved_marker, &recovery_marker) {
Ok(()) if sync => os::fsync_dir_std(&old_path)?,
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
Ok(Ok(Some(old_dir)))
},
)
.await
.map_err(to_file_error)??;
let Some(old_dir) = res else {
return Ok(None);
};
if let Some(uuid) = old_dir {
self.io_backend
.invalidate_cached_fds_under(volume, &format!("{object}/{uuid}"));
}
Ok(Some(()))
}
pub(crate) async fn try_publish_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) || transaction.is_nil() {
return Err(DiskError::InvalidPath);
}
let object_path = self.get_object_path(volume, object)?;
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
let root = self.io_get_object_path(volume, object)?;
let rollback = root.join(transaction.to_string());
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let sync = effective_durability(volume).syncs_commit_metadata();
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
let volume_str = volume.to_owned();
let object_str = object.to_owned();
os::run_blocking_namespace_operation_with_sync(
lease,
sync,
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
Ok(Some(bytes)) => bytes,
Ok(None) => return Ok(Ok(None)),
Err(e) => return Ok(Err(e)),
};
let Ok(journal) = serde_json::from_slice::<DeleteRecoveryJournal>(&prepared) else {
return Ok(Err(DiskError::FileCorrupt));
};
if journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|| journal.volume != volume_str
|| journal.object != object_str
|| journal.transaction != transaction
|| journal.transaction.is_nil()
|| journal.data_dirs.iter().any(|id| id.is_nil() || *id == transaction)
{
return Ok(Err(DiskError::FileCorrupt));
}
let target_satisfied = match journal.target_digest {
None => match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
Ok(exists) => !exists,
Err(e) => return Ok(Err(e)),
},
Some(expected_target) => match regular_bytes(&root.join(STORAGE_FORMAT_FILE)) {
Ok(Some(bytes)) => digest(&bytes) == expected_target,
Ok(None) => false,
Err(e) => return Ok(Err(e)),
},
};
if !target_satisfied {
return Ok(Ok(None));
}
let mut sole_transaction = journal.predecessors.is_empty();
if sole_transaction {
match std::fs::read_dir(&root) {
Ok(entries) => {
for entry in entries {
let entry = entry?;
let Ok(name) = entry.file_name().into_string() else {
sole_transaction = false;
break;
};
if let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil())
&& id != transaction
&& !journal.data_dirs.contains(&id)
{
let dir_path = entry.path();
let is_surviving_data_dir = journal.original_versions.iter().any(|v| v.data_dir == Some(id))
&& !dir_path.join(PREPARED).exists()
&& !dir_path.join(DELETE_QUORUM_COMMITTED_FILE).exists()
&& !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
if !is_surviving_data_dir {
sole_transaction = false;
break;
}
}
}
}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
let receipt = CommitReceipt {
journal: journal.clone(),
finished: false,
};
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
std::fs::create_dir_all(&staging_root)?;
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged.as_file().sync_data()?;
}
std::fs::rename(staged.path(), &receipt_path)?;
if sync {
os::fsync_dir_std(&rollback)?;
}
Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
},
)
.await
.map_err(to_file_error)?
}
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<bool> {
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
match res {
Ok(()) => Ok(()),
Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
Err(err) => Err(err),
}
}
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) || transaction.is_nil() {
return Err(DiskError::InvalidPath);
}
let object_path = self.get_object_path(volume, object)?;
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
let mut claims: Vec<SnapshotLeaseDeleteClaim> = Vec::with_capacity(journal.data_dirs.len());
for data_dir in &journal.data_dirs {
let key = SnapshotLeaseKey {
volume: volume.to_owned(),
path: format!("{object}/{data_dir}"),
};
let opts = DeleteOptions {
recursive: true,
old_data_dir: Some(transaction),
..Default::default()
};
let Some(claim) = self.claim_snapshot_delete(key, SnapshotPendingDelete::Ordinary(opts)) else {
for prior in claims {
prior.finish(false);
}
return Ok(false);
};
claims.push(claim);
}
let bucket_path = self.io_get_bucket_path(volume)?;
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let root = self.io_get_object_path(volume, object)?;
let rollback = root.join(transaction.to_string());
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
let sync = effective_durability(volume).syncs_commit_metadata();
let data_dirs = journal.data_dirs.clone();
let has_target = journal.target_digest.is_some();
let journal_for_receipt = has_target.then(|| journal.clone());
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
let prune_paths: Vec<_> = rollback
.ancestors()
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
.map(Path::to_path_buf)
.collect();
let res = os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<()> {
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
for data_dir in &data_dirs {
for base in [&root, &rollback] {
let dir_path = base.join(data_dir.to_string());
match std::fs::read_dir(&dir_path) {
Ok(entries) => {
let mut removable = Vec::new();
let mut preserve_dir = false;
for entry in entries {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
preserve_dir = true;
continue;
};
if entry.file_type()?.is_file()
&& (delete_recovery_part_file(name)
|| name == committed
|| name == recovery_committed
|| name == reserved
|| is_delete_transaction_marker(name, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX))
{
removable.push(entry.path());
} else {
preserve_dir = true;
}
}
for path in removable {
ignore_not_found(std::fs::remove_file(path))?;
}
if sync {
os::fsync_dir_std(&dir_path)?;
}
if !preserve_dir {
match std::fs::remove_dir(&dir_path) {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound || is_dir_not_empty_error(&err) => {}
Err(err) => return Err(err),
}
}
}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
}
if !data_dirs.is_empty() && sync && root.is_dir() {
os::fsync_dir_std(&root)?;
}
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
}
if let Some(journal) = journal_for_receipt {
let finished_receipt = CommitReceipt { journal, finished: true };
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
std::fs::create_dir_all(&staging_root)?;
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged_receipt.as_file().sync_data()?;
}
std::fs::rename(staged_receipt.path(), &receipt_path)?;
let _ = std::fs::File::create(rollback.join(FINISHED))?;
if sync {
os::fsync_dir_std(&rollback)?;
}
}
for name in [TARGET, PREPARED, FINISHED] {
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
}
if sync {
os::fsync_dir_std(&rollback)?;
}
ignore_not_found(std::fs::remove_file(&receipt_path))?;
for dir_path in &prune_paths {
if let Err(err) = std::fs::remove_dir(dir_path) {
if is_dir_not_empty_error(&err) {
break;
}
if !is_benign_object_rmdir_error(&err) {
return Err(std::io::Error::other(FileAccessDeniedWithContext {
path: dir_path.clone(),
source: err,
}));
}
}
}
Ok(())
})
.await;
let ok = res.is_ok();
for claim in claims {
claim.finish(ok);
}
res.map(|()| true).map_err(|err| to_file_error(err).into())
}
}
#[cfg(not(feature = "e2e-test-hooks"))]
@@ -3582,12 +3038,12 @@ mod tests {
let exit = native.exit_signal.as_ref().expect("fixture cleanup lifecycle");
exit.send(()).expect("stop the existing idle cleanup task");
tokio::time::timeout(Duration::from_secs(5), async {
while exit.receiver_count() != 0 || root_file.strong_count() != 3 {
while exit.receiver_count() != 0 || root_file.strong_count() != 2 {
tokio::task::yield_now().await;
}
})
.await
.expect("only LocalDisk's publication, mount and I/O backend fields may retain the initial root FD");
.expect("only LocalDisk's publication and mount fields may retain the initial root FD");
let transaction = Uuid::new_v4();
let applied = apply(&native, transaction).await;
let committed = native
@@ -3853,171 +3309,6 @@ mod tests {
inspected
}
#[tokio::test]
async fn fused_delete_recovery_survives_interrupted_prepare_and_publication() {
for published in [false, true] {
let (_temp, disk, _info, original) = fixture().await;
let transaction = Uuid::new_v4();
let prepared = disk
.try_fused_recoverable_delete_version(
"recovery",
"object",
&request(),
&DeleteOptions {
recoverable_delete: true,
old_data_dir: Some(transaction),
..Default::default()
},
transaction,
None,
)
.await
.expect("prepare fused deletion");
assert_eq!(prepared, Some(()), "the test must exercise the fused path");
if published {
let (_, sole, target_absent) = disk
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
.await
.expect("publish fused receipt")
.expect("fused publication must apply");
assert!(sole && target_absent);
}
let mut state = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("inspect persisted state after interruption");
if published {
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
for action in [DeleteRecoveryAction::FinishCommit, DeleteRecoveryAction::FinalizeCommit] {
state = disk
.settle_delete_transaction("recovery", "object", transaction, state.fingerprint, action)
.await
.expect("finish published deletion through recovery");
}
assert_eq!(state.state, DeleteRecoveryState::Absent);
assert!(disk.read_all("recovery", "object/xl.meta").await.is_err());
} else {
assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
state = disk
.settle_delete_transaction(
"recovery",
"object",
transaction,
state.fingerprint,
DeleteRecoveryAction::Rollback,
)
.await
.expect("roll back uncommitted fused deletion");
assert_eq!(state.state, DeleteRecoveryState::Absent);
assert_eq!(
disk.read_all("recovery", "object/xl.meta")
.await
.expect("restore original metadata")
.as_ref(),
original
);
}
}
}
#[tokio::test]
async fn fused_delete_recovery_preserves_or_removes_shard_after_interruption() {
use crate::erasure::coding::BitrotWriter;
for mode in 0..3 {
let published = mode != 0;
let (_temp, disk, _info, _) = fixture().await;
let data_dir = Uuid::new_v4();
let payload = Bytes::from_static(b"original shard");
let mut info = test_file_info("object", Uuid::nil(), Some(data_dir), None);
info.size = i64::try_from(payload.len()).expect("payload size");
info.erasure.block_size = payload.len();
info.parts[0].size = payload.len();
info.parts[0].actual_size = info.size;
let original = test_meta(info.clone());
let mut writer = BitrotWriter::new(
std::io::Cursor::new(Vec::new()),
info.erasure.shard_size(),
HashAlgorithm::HighwayHash256S,
);
writer.write(&payload).await.expect("encode shard");
writer.shutdown().await.expect("finish shard encoding");
let encoded = writer.into_inner().into_inner();
let root = disk.io_get_object_path("recovery", "object").expect("object path");
fs::write(root.join(STORAGE_FORMAT_FILE), &original)
.await
.expect("write shard metadata");
let shard_dir = root.join(data_dir.to_string());
fs::create_dir_all(&shard_dir).await.expect("create shard directory");
fs::write(shard_dir.join("part.1"), &encoded)
.await
.expect("write original shard");
let transaction = Uuid::new_v4();
assert_eq!(
disk.try_fused_recoverable_delete_version(
"recovery",
"object",
&request(),
&DeleteOptions {
recoverable_delete: true,
old_data_dir: Some(transaction),
..Default::default()
},
transaction,
None,
)
.await
.expect("prepare fused shard deletion"),
Some(()),
"the test must exercise the fused path"
);
if published {
let (journal, sole, target_absent) = disk
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
.await
.expect("publish fused receipt")
.expect("fused publication must apply");
assert!(sole && target_absent);
if mode == 2 {
assert!(
disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
.await
.expect("finish fused deletion"),
"the fused finalization must apply"
);
assert!(!shard_dir.exists(), "fused finalization must remove the shard");
assert!(!root.join(STORAGE_FORMAT_FILE).exists());
assert!(!root.join(transaction.to_string()).exists());
continue;
}
}
let mut state = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("inspect persisted state");
let actions = if published {
vec![DeleteRecoveryAction::FinishCommit, DeleteRecoveryAction::FinalizeCommit]
} else {
vec![DeleteRecoveryAction::Rollback]
};
for action in actions {
state = disk
.settle_delete_transaction("recovery", "object", transaction, state.fingerprint, action)
.await
.expect("recover shard deletion");
}
assert_eq!(state.state, DeleteRecoveryState::Absent);
if published {
assert!(!shard_dir.exists(), "committed deletion must remove the shard");
} else {
assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("restore metadata"), original);
assert_eq!(fs::read(shard_dir.join("part.1")).await.expect("restore shard"), encoded);
}
}
}
#[tokio::test]
async fn delete_recovery_uncommitted_inline_rolls_back_exact_bytes() {
let (_temp, disk, _info, original) = fixture().await;
@@ -6358,157 +5649,6 @@ mod tests {
b"outside bytes"
);
}
#[tokio::test]
async fn delete_recovery_fused_versioned_delete_marker_and_publish_fast_path() {
let (_temp, disk, info, _) = fixture().await;
let root = disk.io_get_object_path("recovery", "object").expect("object root");
let transaction = Uuid::new_v4();
let mut marker = FileInfo {
name: "object".to_owned(),
version_id: Some(Uuid::new_v4()),
deleted: true,
mark_deleted: true,
mod_time: info.mod_time.map(|t| t + time::Duration::seconds(1)),
..Default::default()
};
let opts = DeleteOptions {
old_data_dir: Some(transaction),
recoverable_delete: true,
..Default::default()
};
let fused = disk
.try_fused_recoverable_delete_version("recovery", "object", &marker, &opts, transaction, None)
.await
.expect("fused delete marker preparation must succeed");
assert_eq!(fused, Some(()), "versioned delete marker must use fused path");
assert!(root.join(transaction.to_string()).join(TARGET).is_file());
assert!(root.join(STORAGE_FORMAT_FILE).is_file());
let published = disk
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
.await
.expect("local publish must succeed")
.expect("versioned target must satisfy local publish");
assert!(published.1, "sole transaction must be true");
assert!(published.2, "target_satisfied must be true");
let finalized = disk
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &published.0, None)
.await
.expect("local finalize must succeed");
assert!(finalized);
assert!(!root.join(transaction.to_string()).exists());
let final_meta = FileMeta::load(&fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("read final xl.meta"))
.expect("decode final xl.meta");
assert_eq!(final_meta.versions.len(), 2);
assert_eq!(final_meta.versions[0].header.version_id, marker.version_id);
marker.deleted = false;
}
#[tokio::test]
async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
let (_temp, disk, mut v1, _) = fixture().await;
let root = disk.io_get_object_path("recovery", "object").expect("object path");
// Configure v1 and v2 with distinct physical data directories so deleting v2
// exercises both `old_dir` reservation and `is_surviving_data_dir`.
v1.data = None;
rustfs_utils::http::metadata_compat::remove_str(
&mut v1.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
);
let v1_dir = Uuid::new_v4();
v1.data_dir = Some(v1_dir);
fs::create_dir_all(root.join(v1_dir.to_string()))
.await
.expect("create v1 dir");
fs::write(root.join(v1_dir.to_string()).join("part.1"), b"v1-data")
.await
.expect("write v1 part");
let mut v2 = v1.clone();
v2.version_id = Some(Uuid::new_v4());
let v2_dir = Uuid::new_v4();
v2.data_dir = Some(v2_dir);
fs::create_dir_all(root.join(v2_dir.to_string()))
.await
.expect("create v2 dir");
fs::write(root.join(v2_dir.to_string()).join("part.1"), b"v2-data")
.await
.expect("write v2 part");
let mut meta = FileMeta::new();
meta.add_version(v1.clone()).expect("add v1");
meta.add_version(v2.clone()).expect("add v2");
fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
.await
.expect("write xl.meta");
let transaction = Uuid::new_v4();
let opts = DeleteOptions {
recoverable_delete: true,
..Default::default()
};
let fused = disk
.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
.await
.expect("fused versioned delete preparation must succeed");
assert_eq!(fused, Some(()));
assert!(
root.join(v2_dir.to_string())
.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"))
.is_file(),
"deleted version data_dir must have recovery marker"
);
let (journal, sole_transaction, target_satisfied) = disk
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
.await
.expect("local publish must succeed")
.expect("versioned target must satisfy local publish");
assert!(sole_transaction, "surviving v1 data_dir must not clear sole_transaction");
assert!(target_satisfied);
assert!(journal.target_digest.is_some());
// Simulate interruption during finish-before-finalize right after TARGET is removed
// (after finished receipt + FINISHED marker were persisted, before receipt_path is removed).
let rollback = root.join(transaction.to_string());
let finished_receipt = CommitReceipt {
journal: journal.clone(),
finished: true,
};
fs::write(
rollback.join(DELETE_QUORUM_COMMITTED_FILE),
serde_json::to_vec(&finished_receipt).expect("serialize finished receipt"),
)
.await
.expect("write finished receipt");
fs::write(rollback.join(FINISHED), []).await.expect("write FINISHED");
fs::remove_file(rollback.join(STORAGE_FORMAT_FILE_BACKUP))
.await
.expect("remove backup");
fs::remove_file(rollback.join(TARGET)).await.expect("remove TARGET");
let inspected = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
.expect("interrupted post-TARGET state must remain inspectable without FileCorrupt");
assert_eq!(inspected.state, DeleteRecoveryState::CommitFinished);
// Complete cleanup via try_finish_and_finalize_prepared_delete_local.
let finalized = disk
.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
.await
.expect("local finalize must succeed after interruption");
assert!(finalized);
assert!(!rollback.exists(), "rollback directory must be pruned");
assert!(!root.join(v2_dir.to_string()).exists(), "deleted v2 data_dir must be removed");
assert!(
root.join(v1_dir.to_string()).join("part.1").is_file(),
"surviving v1 data_dir must remain intact"
);
}
}
#[cfg(test)]
@@ -1323,6 +1323,41 @@ impl LocalDisk {
}
impl SourceStore {
/// Only native source validation may create provenance. Ordinary TMP/heal
/// publication can retain a valid old local anchor, never issue/rebind one.
pub async fn write_retirement_anchor(
&self,
state: &SourceState,
publication: &PublicationIdentity,
directory: &Path,
lease: Arc<os::NamespaceMutationLease>,
) -> Result<()> {
if state.lineage != publication.lineage
|| state.generation != publication.source_generation
|| state.disk_id != self.disk_id
|| state.physical_slot != self.physical_slot
{
return Err(DiskError::ConditionalDeleteConflict);
}
let record = NativeRetirementAnchor {
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
disk_id: self.disk_id,
physical_slot: self.physical_slot,
publication: publication.clone(),
source_original_sha256: state.original_sha256,
source_validation_receipt_id: state.backup_owner,
};
let bytes = encode(&record)?;
let directory = directory.to_path_buf();
let base = self.base.clone();
let root = self.publication_root.clone();
os::run_blocking_namespace_operation(lease, move || {
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
})
.await
.map_err(DiskError::from)
}
/// This caller already owns SourceRW and destination metadata ownership.
/// An Aborting survivor can confirm a historical target only with its
/// independent, same-disk publication anchor. Closure never creates one.
@@ -1534,31 +1569,7 @@ impl LocalDisk {
};
// Copy the small source FI to an independent immutable inode. Its
// canonical inode is never truncated to build the destination proxy.
let directory = staging.private_directory(publication.final_data_dir);
let anchor_record = NativeRetirementAnchor {
protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
disk_id: store.disk_id,
physical_slot: store.physical_slot,
publication: publication.clone(),
source_original_sha256: state.original_sha256,
source_validation_receipt_id: state.backup_owner,
};
let anchor_bytes = encode(&anchor_record)?;
state.validate(store.disk_id, &store.path)?;
let state_raw = encode(&state)?;
let source_dir = store.source.clone();
let base_dir = store.base.clone();
let pub_root = store.publication_root.clone();
let terminal = terminal_record_path(&source_dir, &state);
let original_bytes = original.to_vec();
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes)?;
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)
})
.await
.map_err(DiskError::from)?;
store.write_owned_backup_and_persist(owner, false, original, &state).await?;
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
let context = NativeMultipartPublication {
store,
@@ -1569,6 +1580,12 @@ impl LocalDisk {
validated_prepared: parking_lot::Mutex::new(None),
legacy,
};
let directory = staging.private_directory(context.publication.final_data_dir);
let reserved = context.state.lock().clone();
context
.store
.write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
.await?;
Ok(context)
}
}
@@ -1577,15 +1594,18 @@ impl NativeMultipartPublication {
pub fn canonical_head(&self) -> PathBuf {
self.store.source.join(STORAGE_FORMAT_FILE)
}
#[cfg(test)]
pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
let mut state = self.state.lock().clone();
let current = self.store.load().await?;
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
if encode(current_state)? != encode(&state)? || !current.head_matches_state {
return Err(DiskError::ConditionalDeleteConflict);
}
// Validate the actual persisted representation once, after FileMeta's
// canonicalization. No unvalidated input can install a source proxy.
if !self.store.target_matches(&self.publication, new_metadata)? {
return Err(DiskError::ConditionalDeleteConflict);
}
let expected_state_raw = encode(&state)?;
let validated = ValidatedPreparedPublication {
publication: self.publication.clone(),
sha256: raw_sha(new_metadata),
@@ -1598,109 +1618,17 @@ impl NativeMultipartPublication {
state.prepared_sha256 = Some(validated.sha256);
state.prepared_length = Some(validated.length);
state.advance(MultipartSourcePhase::Prepared)?;
state.validate(self.store.disk_id, &self.store.path)?;
let next_state_raw = encode(&state)?;
let source = self.store.source.clone();
let base = self.store.base.clone();
let root = self.store.publication_root.clone();
let path = self.store.path.clone();
let disk_id = self.store.disk_id;
let physical_slot = self.store.physical_slot;
let disk_ids = self.store.disk_ids.clone();
let backup_owner = state.backup_owner;
let old_bytes = old_metadata.map(|b| b.to_vec());
let terminal = terminal_record_path(&source, &state);
os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
let current_state = current
.state
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_raw = encode(current_state).map_err(IoError::other)?;
if current_raw != expected_state_raw || !current.head_matches_state {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
if let Some(previous) = old_bytes.as_deref() {
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
}
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
})
.await
.map_err(DiskError::from)?;
if let Some(previous) = old_metadata {
self.store
.write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
.await?;
} else {
self.store.persist(&state).await?;
}
*self.state.lock() = state;
*self.validated_prepared.lock() = Some(validated);
Ok(())
}
pub async fn prepare_and_install_head(
&self,
new_metadata: &[u8],
old_metadata: Option<&[u8]>,
dst_file_path: &Path,
directory_guard: &os::RenameCommitGuard,
mutation_lease: Arc<os::NamespaceMutationLease>,
) -> Result<os::PreparedRenameSource> {
let mut state = self.state.lock().clone();
if !self.store.target_matches(&self.publication, new_metadata)? {
return Err(DiskError::ConditionalDeleteConflict);
}
let expected_state_raw = encode(&state)?;
let validated = ValidatedPreparedPublication {
publication: self.publication.clone(),
sha256: raw_sha(new_metadata),
length: new_metadata.len() as u64,
};
if let Some(previous) = old_metadata {
state.previous_target_sha256 = Some(raw_sha(previous));
state.previous_target_length = Some(previous.len() as u64);
}
state.prepared_sha256 = Some(validated.sha256);
state.prepared_length = Some(validated.length);
state.advance(MultipartSourcePhase::Prepared)?;
state.validate(self.store.disk_id, &self.store.path)?;
let next_state_raw = encode(&state)?;
let source = self.store.source.clone();
let base = self.store.base.clone();
let root = self.store.publication_root.clone();
let path = self.store.path.clone();
let disk_id = self.store.disk_id;
let physical_slot = self.store.physical_slot;
let disk_ids = self.store.disk_ids.clone();
let backup_owner = state.backup_owner;
let old_bytes = old_metadata.map(|b| b.to_vec());
let new_bytes = new_metadata.to_vec();
let terminal = terminal_record_path(&source, &state);
let prepared_path = self.prepared_path.clone();
let canonical = self.canonical_head();
let dst_file_path = dst_file_path.to_path_buf();
let owned_directory_guard = directory_guard.clone();
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
let current_state = current
.state
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_raw = encode(current_state).map_err(IoError::other)?;
if current_raw != expected_state_raw || !current.head_matches_state {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
if let Some(previous) = old_bytes.as_deref() {
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
}
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
#[cfg(test)]
super::run_owned_file_before_open(&prepared_path);
let mut prepared_metadata_source =
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
prepared_metadata_source.write_all(&new_bytes, true)?;
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
})
.await
.map_err(DiskError::from)?;
*self.state.lock() = state;
*self.validated_prepared.lock() = Some(validated);
Ok(prepared)
}
#[cfg(test)]
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
{
let guard = self.validated_prepared.lock();
@@ -1773,7 +1701,11 @@ impl NativeMultipartPublication {
}
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
let mut state = self.state.lock().clone();
let expected_state_raw = encode(&state)?;
let current = self.store.load().await?;
let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
if current.head.is_some() || encode(current_state)? != encode(&state)? {
return Err(DiskError::ConditionalDeleteConflict);
}
let (expected_sha256, expected_length) = {
let guard = self.validated_prepared.lock();
let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
@@ -1785,52 +1717,16 @@ impl NativeMultipartPublication {
}
(validated.sha256, validated.length)
};
let target_dir = self.store.target_path(&self.publication)?;
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
state.validate(self.store.disk_id, &self.store.path)?;
let next_state_raw = encode(&state)?;
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
let source = self.store.source.clone();
let base = self.store.base.clone();
let root = self.store.publication_root.clone();
let path = self.store.path.clone();
let disk_id = self.store.disk_id;
let physical_slot = self.store.physical_slot;
let disk_ids = self.store.disk_ids.clone();
let terminal = terminal_record_path(&source, &state);
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
let current_state = current
.state
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_raw = encode(current_state).map_err(IoError::other)?;
if current.head.is_some() || current_raw != expected_state_raw {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
os::read_and_sync_multipart_target_metadata(
&target_dir,
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
&base,
&root,
|target| {
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
Ok(())
},
)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
self.store
.read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
return Err(DiskError::ConditionalDeleteConflict);
}
})?;
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), true)
})
.await
.map_err(DiskError::from)?;
Ok(())
})
.await?;
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
let snapshot = self.store.persist_with_source_sync(&state, true).await?;
*self.state.lock() = state;
let proof = self
.store
@@ -604,7 +604,7 @@ impl LocalDisk {
let publication = os::acquire_namespace_mutation_lease_with_owner(&destination, Some(owner)).await;
let tmp_volume = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let temporary = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &Uuid::new_v4().to_string())?;
self.write_all_internal(&temporary, replacement, SyncMode::FileOnly, &tmp_volume)
self.write_all_internal(&temporary, InternalBuf::Ref(&replacement), SyncMode::FileOnly, &tmp_volume)
.await?;
os::rename_metadata_and_sync_with_lease(
temporary,
+4 -51
View File
@@ -30,21 +30,15 @@ pub mod local;
pub mod multipart_source;
pub mod os;
pub(crate) use io_admission::memory as background_memory;
pub(crate) use io_admission::remote as remote_io_admission;
pub use io_admission::{
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler, PhysicalResourcePlan,
RemoteBackgroundIo, spawn_blocking_with_resource_ownership, with_background_disk_io, with_background_resource_set,
with_background_resource_unit, with_remote_background_io,
};
pub use local::pool_meta_native::{PoolMetaNativeCasCondition, PoolMetaNativeFileIdentity};
#[cfg(test)]
pub(crate) use io_admission::{DiskIoClass, test_resource_units};
pub(crate) use io_admission::DiskIoClass;
pub(crate) use io_admission::{
PhysicalHealBinding, acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, has_resource_io_scope,
with_optional_disk_io_context, with_optional_resource_unit,
acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, with_optional_disk_io_context,
with_optional_resource_unit,
};
pub use io_admission::{with_background_disk_io, with_background_resource_unit};
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
@@ -311,14 +305,6 @@ pub enum Disk {
}
impl Disk {
/// Physical ownership belongs to the node hosting the local attachment.
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
match self {
Self::Local(disk) => Some(disk.physical_admission()),
Self::Remote(_) => None,
}
}
/// A read-only probe; the physical delete repeats the same leaf checks.
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
match self {
@@ -401,39 +387,6 @@ impl Disk {
}
}
pub(crate) async fn try_publish_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: Uuid,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
match self {
Self::Local(disk) => {
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
.await
}
Self::Remote(_) => Ok(None),
}
}
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
&self,
volume: &str,
object: &str,
transaction: Uuid,
journal: &DeleteRecoveryJournal,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<bool> {
match self {
Self::Local(disk) => {
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
.await
}
Self::Remote(_) => Ok(false),
}
}
pub(crate) async fn delete_version_recoverable_with_namespace_owner(
&self,
volume: &str,
+78 -313
View File
@@ -286,11 +286,6 @@ mod dir_fsync_failures {
static FAILURES: LazyLock<Mutex<HashMap<PathBuf, io::ErrorKind>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
#[cfg(target_os = "linux")]
pub(super) fn has_any() -> bool {
!FAILURES.lock().is_empty()
}
pub(super) fn set(dir: &Path, kind: io::ErrorKind) {
FAILURES.lock().insert(dir.to_path_buf(), kind);
}
@@ -328,12 +323,7 @@ pub fn set_local_dir_fsync_failure(disk: &crate::disk::Disk, volume: &str, path:
Ok(())
}
#[cfg(all(target_os = "linux", any(test, feature = "test-util")))]
pub(crate) fn has_dir_fsync_test_failures() -> bool {
dir_fsync_failures::has_any()
}
/// Pause a real native filesystem operation inside its physical executor.
/// Pause a real namespace mutation inside its physical executor.
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
pub(crate) mod prepared_publication_test_hooks {
use super::*;
@@ -343,7 +333,6 @@ pub(crate) mod prepared_publication_test_hooks {
PreparedRename,
Rename,
Remove,
NativeReadOpen,
MetadataReadAfterOpen,
MetadataReadAfterRead,
#[cfg(test)]
@@ -383,17 +372,6 @@ pub(crate) mod prepared_publication_test_hooks {
}
}
pub(crate) fn has_any_hooks() -> bool {
if !BEFORE_PUBLICATION.lock().is_empty() {
return true;
}
#[cfg(test)]
if !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty() {
return true;
}
false
}
#[cfg(test)]
type RenameDestinationHook = Box<dyn FnOnce(&Path) + Send>;
#[cfg(test)]
@@ -481,8 +459,6 @@ pub enum LocalPublicationStage {
PreparedRename,
Rename,
Remove,
/// Pause an accepted native read/open before its original syscall.
NativeReadOpen,
MetadataReadAfterOpen,
MetadataReadAfterRead,
}
@@ -497,7 +473,6 @@ impl LocalPublicationPause {
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
LocalPublicationStage::NativeReadOpen => prepared_publication_test_hooks::Stage::NativeReadOpen,
LocalPublicationStage::MetadataReadAfterOpen => prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
LocalPublicationStage::MetadataReadAfterRead => prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
};
@@ -783,7 +758,7 @@ fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
}
pub(crate) fn file_fdatasync_group_commit_enabled() -> bool {
fn file_fdatasync_group_commit_enabled() -> bool {
#[cfg(test)]
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
return enabled;
@@ -884,13 +859,13 @@ impl OpenedDstDirFsyncGroup {
struct DstDirFsyncWaiter {
namespace_owner: Option<Arc<dyn Send + Sync>>,
resource_unit: Option<super::io_admission::ResourceIoUnit>,
resource_unit: Option<rustfs_concurrency::adaptive::AdaptivePermit>,
result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
}
struct DstDirFsyncOwner {
_namespace: Option<Arc<dyn Send + Sync>>,
_resource: Option<super::io_admission::ResourceIoUnit>,
_resource: Option<rustfs_concurrency::adaptive::AdaptivePermit>,
}
#[derive(Clone)]
@@ -1332,7 +1307,7 @@ type FileFdatasyncGroupKey = (usize, bool);
struct FileFdatasyncWaiter {
files: Vec<PathBuf>,
resource_unit: Option<super::io_admission::ResourceIoUnit>,
resource_unit: Option<rustfs_concurrency::adaptive::AdaptivePermit>,
enqueued_at: Option<Instant>,
wait_role: &'static str,
result_tx: oneshot::Sender<SharedFileFdatasyncResult>,
@@ -2417,10 +2392,6 @@ pub(crate) mod file_sync_probe {
configure(root, None, false)
}
pub(crate) fn is_active() -> bool {
!ROOTS.read().expect("file sync probe lock poisoned").is_empty()
}
pub(super) fn set_failing(root: &Path) -> ProbeGuard {
configure(root, Some(1), false)
}
@@ -2582,25 +2553,18 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
/// itself.
#[cfg_attr(
not(target_os = "linux"),
allow(
dead_code,
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
)
#[allow(
dead_code,
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
)]
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
let dir = dir.as_ref();
let files = regular_files(dir)?;
rustfs_io_metrics::record_put_rename_fdatasync_batch(rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, files.len());
sync_files(&files)?;
let fsync_started = rustfs_io_metrics::put_stage_timer();
let result = fsync_dir_std(dir);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
fsync_started,
);
result
for entry in std::fs::read_dir(dir.as_ref())? {
let entry = entry?;
if entry.file_type()?.is_file() {
sync_file(&entry.path())?;
}
}
fsync_dir_std(dir)
}
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
@@ -2752,114 +2716,22 @@ pub(crate) fn read_dir_probe(path: impl AsRef<Path>, entry_limit: usize) -> io::
})
}
#[cfg(test)]
pub(crate) fn has_non_data_dir_entry(path: impl AsRef<Path>, data_dirs: &[uuid::Uuid]) -> io::Result<bool> {
has_non_data_dir_entry_lazy(path, || data_dirs.to_vec()).map(|(_, res)| res)
}
#[cfg(target_os = "linux")]
pub(crate) fn has_non_data_dir_entry_lazy_from_fd(
dir_fd: &std::fs::File,
load_data_dirs: impl FnOnce() -> Vec<uuid::Uuid>,
) -> io::Result<(Vec<uuid::Uuid>, bool)> {
use rustix::fs::{AtFlags, FileType, RawDir, statat};
let mut raw_buf = [std::mem::MaybeUninit::<u8>::uninit(); 1024];
let mut dir = RawDir::new(dir_fd, &mut raw_buf);
let mut data_dirs: Option<Vec<uuid::Uuid>> = None;
let mut load = Some(load_data_dirs);
let mut valid_entries = 0usize;
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
let Some(entry) = dir.next() else {
return Ok((data_dirs.unwrap_or_default(), false));
};
let entry = match entry {
Ok(entry) => entry,
Err(rustix::io::Errno::NOENT) => return Ok((data_dirs.unwrap_or_default(), false)),
Err(err) => return Err(io::Error::from(err)),
};
let c_name = entry.file_name();
let name_bytes = c_name.to_bytes();
if name_bytes.is_empty() || name_bytes == b"." || name_bytes == b".." {
continue;
}
let file_type = match entry.file_type() {
FileType::Unknown => match statat(dir_fd, c_name, AtFlags::SYMLINK_NOFOLLOW) {
Ok(st) => FileType::from_raw_mode(st.st_mode),
Err(rustix::io::Errno::NOENT) => continue,
Err(err) => return Err(io::Error::from(err)),
},
ft => ft,
};
if file_type == FileType::Directory {
let parsed_uuid = if name_bytes.len() == uuid::fmt::Hyphenated::LENGTH {
std::str::from_utf8(name_bytes).ok().and_then(|s| {
let parsed = uuid::Uuid::try_parse(s).ok()?;
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
(parsed.hyphenated().encode_lower(&mut buf) == s).then_some(parsed)
})
} else {
None
};
let Some(parsed) = parsed_uuid else {
let dirs = data_dirs
.take()
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
return Ok((dirs, true));
};
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
if dirs.binary_search(&parsed).is_err() {
return Ok((std::mem::take(dirs), true));
}
valid_entries += 1;
} else if file_type == FileType::RegularFile {
valid_entries += 1;
} else {
continue;
}
let entry_limit = data_dirs.as_ref().map_or(2, |dirs| dirs.len().saturating_add(2));
if valid_entries == entry_limit {
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
if valid_entries == dirs.len().saturating_add(2) {
return Ok((std::mem::take(dirs), true));
}
}
}
let dirs = data_dirs
.take()
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
Ok((dirs, true))
}
#[cfg(any(not(target_os = "linux"), test))]
pub(crate) fn has_non_data_dir_entry_lazy(
path: impl AsRef<Path>,
load_data_dirs: impl FnOnce() -> Vec<uuid::Uuid>,
) -> io::Result<(Vec<uuid::Uuid>, bool)> {
match has_non_data_dir_entry_lazy_inner(path.as_ref(), load_data_dirs) {
match has_non_data_dir_entry_inner(path.as_ref(), data_dirs) {
Ok(res) => Ok(res),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok((Vec::new(), false)),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false),
Err(err) => Err(err),
}
}
#[cfg(any(not(target_os = "linux"), test))]
fn has_non_data_dir_entry_lazy_inner(
path: &Path,
load_data_dirs: impl FnOnce() -> Vec<uuid::Uuid>,
) -> io::Result<(Vec<uuid::Uuid>, bool)> {
fn has_non_data_dir_entry_inner(path: &Path, data_dirs: &[uuid::Uuid]) -> io::Result<bool> {
let mut dir = std::fs::read_dir(path)?;
let mut data_dirs: Option<Vec<uuid::Uuid>> = None;
let mut load = Some(load_data_dirs);
let entry_limit = data_dirs.len().saturating_add(2);
let mut valid_entries = 0usize;
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
let Some(entry) = dir.next() else {
return Ok((data_dirs.unwrap_or_default(), false));
return Ok(false);
};
let entry = entry?;
let file_name = entry.file_name();
@@ -2869,24 +2741,18 @@ fn has_non_data_dir_entry_lazy_inner(
let file_type = entry.file_type()?;
if file_type.is_dir() {
let parsed_uuid = if file_name.len() == uuid::fmt::Hyphenated::LENGTH {
file_name.to_str().and_then(|s| {
let parsed = uuid::Uuid::try_parse(s).ok()?;
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
(parsed.hyphenated().encode_lower(&mut buf) == s).then_some(parsed)
})
} else {
None
};
let Some(parsed) = parsed_uuid else {
let dirs = data_dirs
.take()
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
return Ok((dirs, true));
};
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
if dirs.binary_search(&parsed).is_err() {
return Ok((std::mem::take(dirs), true));
let is_data_dir = file_name.len() == uuid::fmt::Hyphenated::LENGTH
&& file_name
.to_str()
.and_then(|s| {
let parsed = uuid::Uuid::try_parse(s).ok()?;
let mut buf = [0u8; uuid::fmt::Hyphenated::LENGTH];
(parsed.hyphenated().encode_lower(&mut buf) == s && data_dirs.binary_search(&parsed).is_ok())
.then_some(())
})
.is_some();
if !is_data_dir {
return Ok(true);
}
valid_entries += 1;
} else if file_type.is_file() {
@@ -2895,19 +2761,12 @@ fn has_non_data_dir_entry_lazy_inner(
continue;
}
let entry_limit = data_dirs.as_ref().map_or(2, |dirs| dirs.len().saturating_add(2));
if valid_entries == entry_limit {
let dirs = data_dirs.get_or_insert_with(|| (load.take().expect("load_data_dirs called once"))());
if valid_entries == dirs.len().saturating_add(2) {
return Ok((std::mem::take(dirs), true));
}
return Ok(true);
}
}
let dirs = data_dirs
.take()
.unwrap_or_else(|| (load.take().expect("load_data_dirs called once"))());
Ok((dirs, true))
Ok(true)
}
// read_dir count read limit. when count == 0 unlimit.
@@ -3949,23 +3808,6 @@ pub(crate) async fn rename_all_with_prepared_source(
Ok(())
}
#[cfg(target_os = "linux")]
pub(crate) fn rename_all_with_commit_guard_on_thread(source: &Path, destination: &Path, guard: &RenameCommitGuard) -> Result<()> {
guard.directories.prepare_destination().map_err(to_file_error)?;
#[cfg(test)]
prepared_publication_test_hooks::run_rename_destination(source, destination);
#[cfg(any(test, feature = "test-util"))]
{
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
}
#[cfg(test)]
multipart_source_syscall_failures::check(multipart_source_syscall_failures::Stage::Rename, destination)
.map_err(to_file_error)?;
guard.directories.rename(source, destination, None).map_err(to_file_error)?;
Ok(())
}
#[cfg(not(windows))]
pub(crate) async fn rename_all_with_commit_guard(
src_file_path: impl AsRef<Path>,
@@ -4937,17 +4779,6 @@ impl PublicationRoot {
)
}
#[cfg(target_os = "linux")]
pub(crate) fn open_directory_and_readonly_child(
&self,
dir_path: &Path,
child_name: &str,
) -> io::Result<(std::fs::File, std::fs::File)> {
let dir = self.open_directory(dir_path)?;
let child = open_linux_relative(&dir, Path::new(child_name), rustix::fs::OFlags::RDONLY, rustix::fs::Mode::empty())?;
Ok((dir, child))
}
#[cfg(target_os = "linux")]
fn relative_path<'a>(&self, path: &'a Path) -> io::Result<&'a Path> {
path.strip_prefix(self.io_path.as_path())
@@ -6652,7 +6483,6 @@ pub(crate) fn sync_multipart_owned_record_parents(
if !directory.starts_with(source) {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
}
#[cfg_attr(not(unix), allow(unused_mut))]
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
#[cfg(unix)]
{
@@ -11101,54 +10931,6 @@ mod tests {
assert_eq!(err.kind(), io::ErrorKind::NotFound);
}
#[test]
fn has_non_data_dir_entry_lazy_skips_loader_when_only_xl_meta_exists() {
let temp_dir = tempdir().expect("create temp dir");
std::fs::write(temp_dir.path().join("xl.meta"), b"meta").expect("write xl.meta");
let mut called = 0usize;
let (dirs, has_child) = has_non_data_dir_entry_lazy(temp_dir.path(), || {
called += 1;
vec![uuid::Uuid::new_v4()]
})
.expect("lazy check must succeed");
assert_eq!(called, 0, "inline object dir without subdirs must not invoke loader");
assert!(dirs.is_empty());
assert!(!has_child);
#[cfg(target_os = "linux")]
{
let pub_root = PublicationRoot::new(temp_dir.path()).expect("open pub root");
let (dir_fd, _meta) = pub_root
.open_directory_and_readonly_child(temp_dir.path(), "xl.meta")
.expect("open dir and child");
let mut fd_called = 0usize;
let (fd_dirs, fd_has_child) = has_non_data_dir_entry_lazy_from_fd(&dir_fd, || {
fd_called += 1;
vec![uuid::Uuid::new_v4()]
})
.expect("fd lazy check must succeed");
assert_eq!(fd_called, 0);
assert!(fd_dirs.is_empty());
assert!(!fd_has_child);
}
let data_dir = uuid::Uuid::new_v4();
std::fs::create_dir(temp_dir.path().join(data_dir.to_string())).expect("create data dir");
let (dirs, has_child) = has_non_data_dir_entry_lazy(temp_dir.path(), || {
called += 1;
vec![data_dir]
})
.expect("lazy check with data dir must succeed");
assert_eq!(called, 1, "data dir presence must invoke loader once");
assert_eq!(dirs, vec![data_dir]);
assert!(!has_child);
std::fs::create_dir(temp_dir.path().join("nested-key")).expect("create nested child dir");
let (dirs, has_child) = has_non_data_dir_entry_lazy(temp_dir.path(), || vec![data_dir]).expect("lazy check child");
assert_eq!(dirs, vec![data_dir], "child presence must still return data_dirs for skip set");
assert!(has_child);
}
#[tokio::test]
#[serial_test::serial(file_sync_probe)]
async fn resource_grouped_fdatasync_retired_limiter_keeps_its_budget_until_worker_cleanup() {
@@ -11318,71 +11100,54 @@ mod tests {
#[tokio::test]
async fn resource_native_dispatch_retains_each_unit_after_caller_cancellation() {
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
use crate::disk::io_admission::with_background_resource_unit;
use rustfs_concurrency::adaptive::AdaptiveAdmission;
use std::num::NonZeroU16;
let mut retained = Vec::new();
for physical_count in [1, 2] {
for lane in [
None,
Some(DiskBlockingLane::Foreground),
Some(DiskBlockingLane::Isolated),
Some(DiskBlockingLane::Fsync),
] {
let (unit, gates) = test_resource_units(physical_count);
let (started_tx, started_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let (finished_tx, finished_rx) = oneshot::channel();
let worker = tokio::spawn(with_optional_resource_unit(Some(unit), async move {
let operation = move || {
started_tx.send(()).expect("native operation entered");
let _ = release_rx.blocking_recv();
let _ = finished_tx.send(());
Ok(())
};
match lane {
Some(lane) => {
DiskBlockingQueue::new()
.run_on_lane(
lane,
"native operation failed",
DiskBlockingQueue::test_stages_for_lane(lane),
move |_| operation(),
)
.await
}
None => fsync_spawn_blocking(operation).await.expect("native operation joined"),
for lane in [None, Some(DiskBlockingLane::Foreground), Some(DiskBlockingLane::Fsync)] {
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
let unit = gate.acquire().await;
let (started_tx, started_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let (finished_tx, finished_rx) = oneshot::channel();
let worker = tokio::spawn(with_background_resource_unit(unit, async move {
let operation = move || {
started_tx.send(()).expect("native operation entered");
let _ = release_rx.blocking_recv();
let _ = finished_tx.send(());
Ok(())
};
match lane {
Some(lane) => {
DiskBlockingQueue::new()
.run_on_lane(
lane,
"native operation failed",
DiskBlockingQueue::test_stages_for_lane(lane),
move |_| operation(),
)
.await
}
}));
started_rx.await.expect("native operation started");
worker.abort();
assert!(worker.await.expect_err("caller cancelled").is_cancelled());
retained.push(gates.iter().map(|gate| gate.snapshot().active).collect::<Vec<_>>());
// Drain native work before assertions, including in the negative control.
release_tx.send(()).expect("release native operation");
finished_rx.await.expect("native operation finished");
tokio::time::timeout(Duration::from_secs(1), async {
while gates.iter().any(|gate| gate.snapshot().active != 0) {
tokio::task::yield_now().await;
}
})
.await
.expect("native completion returns the unit");
}
None => fsync_spawn_blocking(operation).await.expect("native operation joined"),
}
}));
started_rx.await.expect("native operation started");
worker.abort();
assert!(worker.await.expect_err("caller cancelled").is_cancelled());
retained.push(gate.snapshot().active);
// Drain native work before assertions, including in the negative control.
release_tx.send(()).expect("release native operation");
finished_rx.await.expect("native operation finished");
tokio::time::timeout(Duration::from_secs(1), async {
while gate.snapshot().active != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("native completion returns the unit");
}
assert_eq!(
retained,
[
vec![1],
vec![1],
vec![1],
vec![1],
vec![1, 1],
vec![1, 1],
vec![1, 1],
vec![1, 1]
],
"native dispatch and all queue lanes must retain their own producer's units"
);
assert_eq!(retained, [1, 1, 1], "fsync and both queue lanes must retain their own producer's unit");
}
#[tokio::test]
+4 -4
View File
@@ -76,14 +76,14 @@ impl DriverReadBudget {
}
}
pub(super) fn start_driver<O: Send + Unpin + 'static>(
pub(super) fn start_driver(
&self,
entries: u32,
shards: usize,
) -> Result<rustfs_uring::OwnedUringDriver<O>, rustfs_uring::ProbeFailure> {
) -> Result<rustfs_uring::UringDriver, rustfs_uring::ProbeFailure> {
match self {
Self::Disabled => rustfs_uring::OwnedUringDriver::probe_and_start_sharded(entries, shards),
Self::Limited { pool, per_driver } => rustfs_uring::OwnedUringDriver::probe_and_start_with_shared_budget(
Self::Disabled => rustfs_uring::UringDriver::probe_and_start_sharded(entries, shards),
Self::Limited { pool, per_driver } => rustfs_uring::UringDriver::probe_and_start_with_shared_budget(
entries,
shards,
rustfs_uring::ReadLimits {
+25 -423
View File
@@ -98,9 +98,6 @@ pin_project! {
// contiguous on-disk `[hash][data]` block so both are pulled in a single
// pass; grown lazily and never shrunk.
buf: Vec<u8>,
// Free scratch storage before refunding its allocation.
buf_memory: Option<crate::disk::background_memory::Reservation>,
memory_budget: Option<std::sync::Arc<crate::disk::BackgroundMemoryBudget>>,
// Reused owned chunk vector for the remote HTTP fast path. Keeping the
// allocation with the reader avoids allocating once per bitrot block.
chunks: Vec<bytes::Bytes>,
@@ -121,8 +118,6 @@ where
hash_algo: algo,
shard_size,
buf: Vec::new(),
buf_memory: None,
memory_budget: crate::disk::current_resource_unit().and_then(|unit| unit.memory_budget()),
chunks: Vec::new(),
skip_verify,
last_verify_duration: Duration::ZERO,
@@ -214,7 +209,6 @@ where
/// On `Ok` the block is `self.buf[..need]`; the caller re-borrows it so the
/// verification can also borrow `self` immutably.
async fn read_scratch_block(&mut self, need: usize, want: usize) -> std::io::Result<()> {
reserve_bitrot_scratch(&mut self.buf, &mut self.buf_memory, self.memory_budget.as_ref(), need)?;
if self.buf.len() < need {
self.buf.resize(need, 0);
}
@@ -235,28 +229,6 @@ where
}
}
// Scratch growth happens only before a complete block or an empty chunk tail.
// Old contents are consumed; keep both allocations charged until replacement.
fn reserve_bitrot_scratch(
buf: &mut Vec<u8>,
reservation: &mut Option<crate::disk::background_memory::Reservation>,
budget: Option<&std::sync::Arc<crate::disk::BackgroundMemoryBudget>>,
capacity: usize,
) -> std::io::Result<()> {
if let Some(budget) = budget
&& buf.capacity() < capacity
{
let memory = budget.reserve(capacity)?;
let mut replacement = Vec::new();
replacement
.try_reserve_exact(capacity)
.map_err(|_| std::io::Error::from(std::io::ErrorKind::OutOfMemory))?;
*buf = replacement;
*reservation = Some(memory);
}
Ok(())
}
/// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
fn short_shard_read(got: usize, want: usize) -> std::io::Error {
error!(
@@ -367,8 +339,6 @@ where
let inner = &mut self.inner;
let chunks = &mut self.chunks;
let tail_buf = &mut self.buf;
let tail_memory = &mut self.buf_memory;
let memory_budget = self.memory_budget.as_ref();
let mut received = 0usize;
poll_fn(|cx| {
for _ in 0..MAX_CHUNK_POLLS_PER_YIELD {
@@ -411,9 +381,7 @@ where
if chunks.len() == MAX_RETAINED_CHUNKS_PER_BLOCK {
if tail_buf.is_empty() {
let tail_capacity = need - (received - chunk.len());
reserve_bitrot_scratch(tail_buf, tail_memory, memory_budget, tail_capacity)?;
tail_buf.reserve_exact(tail_capacity);
tail_buf.reserve_exact(need - (received - chunk.len()));
}
tail_buf.extend_from_slice(&chunk);
} else {
@@ -689,12 +657,7 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
algo: HashAlgorithm,
mut shard_size: usize,
) -> std::io::Result<()> {
// Every supported digest fits on the stack; only shard payloads need leases.
let mut hash = [0; 64];
let hash_buf = &mut hash[..algo.size()];
let mut background_buffer = crate::disk::current_resource_unit()
.and_then(|unit| unit.memory_budget())
.map(|budget| crate::disk::background_memory::BackgroundBuffer::new(Some(budget)));
let mut hash_buf = vec![0; algo.size()];
let mut left = want_size;
if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) {
@@ -702,30 +665,21 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
}
while left > 0 {
let n = r.read_exact(hash_buf).await?;
let n = r.read_exact(&mut hash_buf).await?;
left -= n;
if left < shard_size {
shard_size = left;
}
let mut pooled = None;
let buf = if let Some(buffer) = background_buffer.as_mut() {
buffer.ensure_len(shard_size)?;
&mut buffer.as_mut_slice()[..shard_size]
} else {
let buffer = pooled.insert(get_ec_buffer(shard_size));
buffer.resize(shard_size, 0);
buffer.as_mut_slice()
};
let read = r.read_exact(buf).await?;
let mut buf = get_ec_buffer(shard_size);
buf.resize(shard_size, 0);
let read = r.read_exact(&mut buf).await?;
let actual_hash = algo.hash_encode(buf);
let actual_hash = algo.hash_encode(&buf);
let hash_ok = actual_hash.as_ref() == &hash_buf[0..n];
drop(actual_hash); // 释放借用
if let Some(buf) = pooled {
return_ec_buffer(buf);
}
return_ec_buffer(buf);
if !hash_ok {
return Err(std::io::Error::other("bitrot hash mismatch"));
}
@@ -744,74 +698,15 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
/// Custom writer enum that supports inline buffer storage
pub enum CustomWriter {
/// Inline buffer writer - stores data in memory
InlineBuffer(InlineBuffer),
InlineBuffer(Vec<u8>),
/// Disk-based writer using tokio file
Other(Box<dyn AsyncWrite + Unpin + Send + Sync>),
}
/// Inline payload storage retains its background allocation through export.
pub struct InlineBuffer {
foreground: Vec<u8>,
background: Option<crate::disk::background_memory::BackgroundBuffer>,
filled: usize,
}
impl InlineBuffer {
fn new() -> Self {
let background = crate::disk::current_resource_unit()
.and_then(|unit| unit.memory_budget())
.map(|budget| crate::disk::background_memory::BackgroundBuffer::new(Some(budget)));
Self {
foreground: Vec::new(),
background,
filled: 0,
}
}
fn as_slice(&self) -> &[u8] {
match &self.background {
Some(buffer) => &buffer.as_slice()[..self.filled],
None => &self.foreground,
}
}
fn append(&mut self, bufs: &[IoSlice<'_>]) -> std::io::Result<usize> {
let total = bufs
.iter()
.try_fold(0usize, |length, buf| length.checked_add(buf.len()).ok_or(std::io::ErrorKind::OutOfMemory))?;
if let Some(buffer) = &mut self.background {
let length = self.filled.checked_add(total).ok_or(std::io::ErrorKind::OutOfMemory)?;
if length > buffer.as_slice().len() {
let capacity = buffer.as_slice().len().checked_mul(2).unwrap_or(length).max(length);
// Admission includes both allocations until the prefix has been copied.
buffer.grow(capacity, self.filled)?;
}
let mut offset = self.filled;
for buf in bufs {
buffer.as_mut_slice()[offset..offset + buf.len()].copy_from_slice(buf);
offset += buf.len();
}
self.filled = length;
} else {
for buf in bufs {
self.foreground.extend_from_slice(buf);
}
}
Ok(total)
}
fn into_bytes(self) -> bytes::Bytes {
match self.background {
Some(mut buffer) => buffer.take_bytes(self.filled),
None => bytes::Bytes::from(self.foreground),
}
}
}
impl CustomWriter {
/// Create a new inline buffer writer
pub fn new_inline_buffer() -> Self {
Self::InlineBuffer(InlineBuffer::new())
Self::InlineBuffer(Vec::new())
}
/// Create a new disk writer from any AsyncWrite implementation
@@ -825,15 +720,15 @@ impl CustomWriter {
/// Get the inline buffer data if this is an inline buffer writer
pub fn get_inline_data(&self) -> Option<&[u8]> {
match self {
Self::InlineBuffer(data) => Some(data.as_slice()),
Self::InlineBuffer(data) => Some(data),
Self::Other(_) => None,
}
}
/// Extract the inline buffer data, consuming the writer
pub fn into_inline_data(self) -> Option<bytes::Bytes> {
pub fn into_inline_data(self) -> Option<Vec<u8>> {
match self {
Self::InlineBuffer(data) => Some(data.into_bytes()),
Self::InlineBuffer(data) => Some(data),
Self::Other(_) => None,
}
}
@@ -846,7 +741,10 @@ impl AsyncWrite for CustomWriter {
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
match self.get_mut() {
Self::InlineBuffer(data) => std::task::Poll::Ready(data.append(&[IoSlice::new(buf)])),
Self::InlineBuffer(data) => {
data.extend_from_slice(buf);
std::task::Poll::Ready(Ok(buf.len()))
}
Self::Other(writer) => {
let pinned_writer = std::pin::Pin::new(writer.as_mut());
pinned_writer.poll_write(cx, buf)
@@ -880,7 +778,13 @@ impl AsyncWrite for CustomWriter {
bufs: &[IoSlice<'_>],
) -> std::task::Poll<std::io::Result<usize>> {
match self.get_mut() {
Self::InlineBuffer(data) => std::task::Poll::Ready(data.append(bufs)),
Self::InlineBuffer(data) => {
let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
for buf in bufs {
data.extend_from_slice(buf);
}
std::task::Poll::Ready(Ok(total))
}
Self::Other(writer) => {
let pinned_writer = std::pin::Pin::new(writer.as_mut());
pinned_writer.poll_write_vectored(cx, bufs)
@@ -955,7 +859,7 @@ impl BitrotWriterWrapper {
}
/// Extract the inline buffer data, consuming the wrapper
pub fn into_inline_data(self) -> Option<bytes::Bytes> {
pub fn into_inline_data(self) -> Option<Vec<u8>> {
match self.writer_type {
WriterType::InlineBuffer => {
let writer = self.bitrot_writer.into_inner();
@@ -1602,184 +1506,6 @@ mod tests {
assert_eq!(data, &out[..]);
}
#[tokio::test]
async fn bitrot_reader_scratch_budget_preserves_growth_and_reuse() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
let algo = HashAlgorithm::HighwayHash256S;
let mut encoded = Vec::new();
for data in [&b"abcd"[..], &b"abcdefgh"[..], &b"ijklmnop"[..]] {
encoded.extend_from_slice(algo.hash_encode(data).as_ref());
encoded.extend_from_slice(data);
}
let memory = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
let mut reader = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BitrotReader::new(Cursor::new(encoded), 8, algo, false)
})
.await;
let mut first = [0; 4];
assert_eq!(
reader.read(&mut first).await.expect_err("closed scratch budget").kind(),
io::ErrorKind::WouldBlock
);
assert_eq!(reader.inner.position(), 0, "allocation denial must not consume a frame");
memory.set_sampled_limit(36, Duration::from_secs(30));
reader.read(&mut first).await.expect("first scratch allocation fits");
assert_eq!(&first, b"abcd");
assert_eq!(memory.snapshot().used, 36);
memory.set_sampled_limit(75, Duration::from_secs(30));
let mut next = [0; 8];
assert_eq!(
reader
.read(&mut next)
.await
.expect_err("growth requires old36 plus new40")
.kind(),
io::ErrorKind::WouldBlock
);
assert_eq!(reader.inner.position(), 36);
assert_eq!(memory.snapshot().used, 36);
memory.set_sampled_limit(76, Duration::from_secs(30));
reader.read(&mut next).await.expect("growth now fits");
assert_eq!(&next, b"abcdefgh");
assert_eq!(memory.snapshot().used, 40);
memory.set_sampled_limit(0, Duration::ZERO);
reader.read(&mut next).await.expect("reuse after hot shrink");
assert_eq!(&next, b"ijklmnop");
assert_eq!(memory.snapshot().used, 40);
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn bitrot_chunk_tail_allocation_stays_charged_until_reader_drop() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
let algo = HashAlgorithm::HighwayHash256S;
let payload = vec![37; 256];
let encoded = encode_one_block(&payload, payload.len(), algo.clone()).await;
let tail_capacity = encoded.len() - MAX_RETAINED_CHUNKS_PER_BLOCK;
for fits in [false, true] {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(if fits { tail_capacity as u64 } else { 0 }, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut reader = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
BitrotReader::new(GeneratedChunkSource::new(encoded.clone(), 1), payload.len(), algo.clone(), false)
})
.await;
let mut output = Vec::with_capacity(payload.len());
let result = reader.read_appending(&mut output, payload.len()).await;
if fits {
assert_eq!(result.expect("accounted tail fits"), payload.len());
assert_eq!(output, payload);
assert_eq!(memory.snapshot().used, tail_capacity as u64);
memory.set_sampled_limit(0, Duration::ZERO);
assert_eq!(memory.snapshot().used, tail_capacity as u64);
} else {
assert_eq!(result.expect_err("closed tail budget").kind(), io::ErrorKind::WouldBlock);
assert!(output.is_empty(), "an unverified fragment must not reach output");
assert_eq!(memory.snapshot().used, 0);
}
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn bitrot_verify_background_memory_is_bounded_and_reused() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
struct ShrinkingReader {
data: Cursor<Vec<u8>>,
memory: Arc<BackgroundMemoryBudget>,
first_block_end: u64,
peak: Arc<std::sync::atomic::AtomicU64>,
}
impl AsyncRead for ShrinkingReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
self.peak.fetch_max(self.memory.snapshot().used, Ordering::SeqCst);
let result = Pin::new(&mut self.data).poll_read(cx, out);
if self.data.position() >= self.first_block_end {
self.memory.set_sampled_limit(0, Duration::ZERO);
}
result
}
}
for algo in [
HashAlgorithm::None,
HashAlgorithm::HighwayHash256S,
HashAlgorithm::HighwayHash256SLegacy,
] {
let payload = [37; 17];
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), 8, algo.clone());
for chunk in payload.chunks(8) {
writer.write(chunk).await.expect("encoded bitrot block");
}
writer.shutdown().await.expect("complete fixture");
let encoded = writer.into_inner().into_inner();
let memory = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
let error = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
bitrot_verify(Cursor::new(&encoded), encoded.len(), payload.len(), algo.clone(), 8),
)
.await
.expect_err("a closed background budget must reject bitrot payload allocation");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(memory.snapshot().used, 0);
memory.set_sampled_limit(8, Duration::from_secs(30));
let peak = Arc::new(std::sync::atomic::AtomicU64::new(0));
let reader = ShrinkingReader {
data: Cursor::new(encoded.clone()),
memory: memory.clone(),
first_block_end: (algo.size() + 8) as u64,
peak: peak.clone(),
};
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
bitrot_verify(reader, encoded.len(), payload.len(), algo, 8),
)
.await
.expect("reuse the same allocation after shrink, including the short tail");
assert_eq!(peak.load(Ordering::SeqCst), 8);
assert_eq!(memory.snapshot().limit, 0);
assert_eq!(
memory.snapshot().used,
0,
"the verifier frees its payload instead of retaining it in an uncharged pool"
);
}
}
#[tokio::test]
async fn bitrot_verify_cancelled_payload_read_refunds_memory() {
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
let algo = HashAlgorithm::HighwayHash256S;
let (mut sender, reader) = tokio::io::duplex(64);
sender
.write_all(algo.hash_encode(&[37; 8]).as_ref())
.await
.expect("send only hash");
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let mut verify = Box::pin(with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
bitrot_verify(reader, algo.size() + 8, 8, algo, 8),
));
assert!(futures::poll!(&mut verify).is_pending(), "the payload read waits after the complete hash");
let retained = memory.snapshot().used;
memory.set_sampled_limit(0, Duration::ZERO);
drop(verify);
drop(sender);
assert_eq!(retained, 8, "pending verification owns its payload allocation");
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn bitrot_verify_accepts_valid_shard_file_and_rejects_size_or_hash_mismatch() {
let data = b"bitrot verify covers every shard";
@@ -2677,128 +2403,4 @@ mod tests {
assert_eq!(output, vec![9u8]);
}
}
mod inline_memory_regressions {
use super::*;
use crate::disk::background_memory::is_memory_pressure;
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
use std::sync::Arc;
use std::time::Duration;
use tokio::io::AsyncWriteExt;
async fn writer_for(memory: Arc<BackgroundMemoryBudget>) -> CustomWriter {
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(memory)), async { CustomWriter::new_inline_buffer() })
.await
}
#[tokio::test]
async fn inline_writer_rejects_closed_and_expired_owners_before_first_allocation() {
for expired in [false, true] {
let memory = BackgroundMemoryBudget::new();
if expired {
memory.set_sampled_limit(128, Duration::ZERO);
}
let mut writer = writer_for(memory.clone()).await;
let error = writer.write_all(&[7; 64]).await.expect_err("no live grant");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert!(is_memory_pressure(&error));
assert_eq!(writer.get_inline_data(), Some(&[][..]));
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn inline_writer_growth_charges_old_plus_new_and_is_atomic_for_vectored_writes() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(128, Duration::from_secs(30));
let mut writer = writer_for(memory.clone()).await;
writer.write_all(&[1; 64]).await.expect("first allocation");
assert_eq!(memory.snapshot().used, 64);
let second = [2; 32];
let third = [3; 32];
let chunks = [IoSlice::new(&second), IoSlice::new(&third)];
let error = writer.write_vectored(&chunks).await.expect_err("64 old + 128 new exceed 128");
assert!(is_memory_pressure(&error));
assert_eq!(writer.get_inline_data(), Some(&[1; 64][..]));
assert_eq!(memory.snapshot().used, 64);
memory.set_sampled_limit(192, Duration::from_secs(30));
assert_eq!(writer.write_vectored(&chunks).await.expect("old and new both fit"), 64);
assert_eq!(memory.snapshot().used, 128);
let data = writer.get_inline_data().expect("inline payload");
assert_eq!(&data[..64], &[1; 64]);
assert_eq!(&data[64..96], &second);
assert_eq!(&data[96..], &third);
drop(writer);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn inline_writer_expiry_preserves_charged_storage_but_blocks_growth() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(192, Duration::from_secs(30));
let mut writer = writer_for(memory.clone()).await;
writer.write_all(&[5; 64]).await.expect("existing storage");
memory.set_sampled_limit(192, Duration::ZERO);
let error = writer
.write_all(&[9])
.await
.expect_err("expired sample cannot authorize replacement");
assert!(is_memory_pressure(&error));
assert_eq!(writer.get_inline_data(), Some(&[5; 64][..]));
assert_eq!(memory.snapshot().used, 64);
writer.write_all(&[]).await.expect("empty write needs no allocation");
writer.flush().await.expect("flush does not revoke storage");
writer.shutdown().await.expect("shutdown does not revoke storage");
let bytes = writer.into_inline_data().expect("export after expiry");
assert_eq!(bytes.as_ref(), &[5; 64]);
assert_eq!(memory.snapshot().used, 64);
drop(bytes);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn inline_writer_export_keeps_receipt_through_clones_and_the_last_slice() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(64, Duration::from_secs(30));
let mut writer = writer_for(memory.clone()).await;
writer.write_all(&[11; 64]).await.expect("allocate payload");
let bytes = writer.into_inline_data().expect("export bytes");
let clone = bytes.clone();
let slice = bytes.slice(3..7);
memory.set_sampled_limit(0, Duration::ZERO);
drop(bytes);
drop(clone);
assert_eq!(slice.as_ref(), &[11; 4]);
assert_eq!(memory.snapshot().used, 64);
drop(slice);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn inline_writer_keeps_the_constructing_owner_after_leaving_its_scope() {
let first = BackgroundMemoryBudget::new();
first.set_sampled_limit(64, Duration::from_secs(30));
let mut writer = writer_for(first.clone()).await;
let second = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(second.clone())), async {
writer.write_all(&[13; 64]).await.expect("writer still owns the first grant");
})
.await;
assert_eq!(first.snapshot().used, 64);
assert_eq!(second.snapshot().used, 0);
drop(writer);
assert_eq!(first.snapshot().used, 0);
}
#[tokio::test]
async fn foreground_inline_writer_preserves_payload_and_empty_writes() {
let mut writer = CustomWriter::new_inline_buffer();
writer.write_all(&[]).await.expect("empty foreground write");
writer.write_all(b"inline body").await.expect("foreground write");
let bytes = writer.into_inline_data().expect("foreground export");
assert_eq!(bytes.as_ref(), b"inline body");
}
}
}
+6 -23
View File
@@ -175,10 +175,8 @@ const DEFAULT_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE: bool = false;
/// stripe via their stripe handle — only when a data shard is missing or dies.
/// This halves per-GET read bytes, IOPS, and bitrot hashing on healthy
/// objects with 2+2 layouts.
/// Default: false for raw `Erasure::decode` calls without deferred parity
/// handles; `DataBlocksFirst` reader setup attaches deferred parity handles and
/// leaves those parity reserves unengaged by default unless this env var is set
/// to `false`.
/// Default: false (current behavior: every live shard reader is read on every
/// stripe).
const ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE: &str = "RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE";
const DEFAULT_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE: bool = false;
@@ -191,13 +189,6 @@ pub(crate) fn get_lockstep_data_shards_only_enabled() -> bool {
)
}
/// Whether `DataBlocksFirst` reader setup and `with_deferred_parity_handles`
/// should keep parity shards deferred until needed on healthy GETs.
pub(crate) fn get_lockstep_data_shards_only_with_deferred_parity_enabled() -> bool {
matches!(decode_read_policy(), DecodeReadPolicy::DemandBound)
|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, true)
}
/// Get whether bitrot-decode overlap is enabled.
fn is_bitrot_decode_overlap_enabled() -> bool {
rustfs_utils::get_env_bool(
@@ -841,13 +832,6 @@ where
pub(crate) fn with_deferred_parity_handles(mut self, mut handles: Vec<Option<DeferredReaderStripeHandle>>) -> Self {
handles.resize_with(self.readers.len(), || None);
handles.truncate(self.readers.len());
if get_lockstep_data_shards_only_with_deferred_parity_enabled() {
for (index, handle) in handles.iter().enumerate().skip(self.data_shards) {
if handle.is_some() {
self.engaged[index] = false;
}
}
}
self.deferred_handles = handles;
self
}
@@ -2612,8 +2596,7 @@ impl Erasure {
// the ordinary GET read quorum. Only the opt-in GET rollout requires a
// surplus source; available surplus shards and integrity proofs are
// still verified before any reconstructed copy bytes are emitted.
let require_surplus_source =
get_lockstep_data_shards_only_enabled() && matches!(decode_read_policy(), DecodeReadPolicy::Default);
let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
let start = offset / self.block_size;
let end = end_offset.saturating_sub(1) / self.block_size;
@@ -4384,9 +4367,9 @@ mod tests {
.await;
}
/// Compatibility lock: with the gate off (default), raw `Erasure::decode`
/// without deferred parity handles keeps the pre-backlog#923 behavior and
/// reads every live shard — including parity — on every stripe.
/// Compatibility lock: with the gate off (default), the lockstep path
/// keeps the pre-backlog#923 behavior and reads every live shard —
/// including parity — on every stripe.
#[tokio::test]
#[serial_test::serial]
async fn test_lockstep_default_reads_all_shards_per_stripe() {
+35 -209
View File
@@ -16,8 +16,6 @@
//! Supports legacy (reed-solomon-simd) for reading/healing old-version files.
//!
use crate::disk::background_memory::{BackgroundBuffer, Reservation};
use crate::disk::{BackgroundMemoryBudget, current_resource_unit};
use bytes::{Bytes, BytesMut};
use reed_solomon_simd;
use rustfs_erasure_codec::galois_8::ReedSolomon;
@@ -146,54 +144,12 @@ pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize {
/// Reed-Solomon encoder for legacy (main branch) format using reed-solomon-simd.
/// Used when decoding/encoding files with uses_legacy_checksum == true.
struct LegacyWorkspace<T> {
codec: T,
// The cached payload is freed before its reservation.
memory: Option<Reservation>,
payload_bound: usize,
}
impl<T> LegacyWorkspace<T> {
fn prepare(
cached: Option<Self>,
payload_bound: usize,
budget: Option<&Arc<BackgroundMemoryBudget>>,
create: impl FnOnce() -> io::Result<T>,
reset: impl FnOnce(&mut T) -> io::Result<()>,
) -> io::Result<Self> {
if let Some(mut cached) = cached {
let reusable = match (&cached.memory, budget) {
(None, None) => true,
(Some(memory), budget) => {
payload_bound <= cached.payload_bound && budget.is_none_or(|budget| memory.belongs_to(budget))
}
// Uncharged foreground caches can have larger retained Vec
// capacities after reset; start a measured background workspace.
(None, Some(_)) => false,
};
if reusable && reset(&mut cached.codec).is_ok() {
return Ok(cached);
}
// A fresh allocation avoids hidden geometric Vec growth. No payload
// references escape codec calls, so dropping this cache is safe.
drop(cached);
}
let memory = budget.map(|budget| budget.reserve(payload_bound)).transpose()?;
let codec = create()?;
Ok(Self {
codec,
memory,
payload_bound,
})
}
}
struct LegacyReedSolomonEncoder {
data_shards: usize,
parity_shards: usize,
cache_workspaces: bool,
encoder_cache: RwLock<Option<LegacyWorkspace<reed_solomon_simd::ReedSolomonEncoder>>>,
decoder_cache: RwLock<Option<LegacyWorkspace<reed_solomon_simd::ReedSolomonDecoder>>>,
encoder_cache: RwLock<Option<reed_solomon_simd::ReedSolomonEncoder>>,
decoder_cache: RwLock<Option<reed_solomon_simd::ReedSolomonDecoder>>,
}
impl LegacyReedSolomonEncoder {
@@ -243,29 +199,20 @@ impl LegacyReedSolomonEncoder {
.write()
.map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))?
.take();
let budget = current_resource_unit().and_then(|resources| resources.memory_budget());
// reed-solomon-simd 3.1.0 Shards::resize allocates 64-byte units. A
// fresh Vec has at least four units; the decoder work-count bound also
// covers both encoder rate strategies.
let payload_bound = self
.logical_shard_bytes_upper_bound(shard_len)
.ok_or(io::ErrorKind::OutOfMemory)?
.max(256);
let mut workspace = LegacyWorkspace::prepare(
cached_encoder,
payload_bound,
budget.as_ref(),
|| {
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))
},
|encoder| {
encoder
.reset(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to reset SIMD encoder: {e:?}")))
},
)?;
let encoder = &mut workspace.codec;
let mut encoder = {
match cached_encoder {
Some(mut cached) => {
if cached.reset(self.data_shards, self.parity_shards, shard_len).is_err() {
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?
} else {
cached
}
}
None => reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?,
}
};
for (i, shard) in shards.iter().enumerate().take(self.data_shards) {
encoder
.add_original_shard(shard)
@@ -286,7 +233,7 @@ impl LegacyReedSolomonEncoder {
.write()
.map_err(|_| io::Error::other("Failed to return encoder to cache"))?;
if cache.is_none() {
*cache = Some(workspace);
*cache = Some(encoder);
}
}
Ok(())
@@ -307,26 +254,21 @@ impl LegacyReedSolomonEncoder {
.write()
.map_err(|_| io::Error::other("Failed to acquire decoder cache lock"))?
.take();
let budget = current_resource_unit().and_then(|resources| resources.memory_budget());
let payload_bound = self
.logical_shard_bytes_upper_bound(shard_len)
.ok_or(io::ErrorKind::OutOfMemory)?
.max(256);
let mut workspace = LegacyWorkspace::prepare(
cached_decoder,
payload_bound,
budget.as_ref(),
|| {
reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))
},
|decoder| {
decoder
.reset(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to reset SIMD decoder: {e:?}")))
},
)?;
let decoder = &mut workspace.codec;
let mut decoder = {
match cached_decoder {
Some(mut cached_decoder) => {
if let Err(e) = cached_decoder.reset(self.data_shards, self.parity_shards, shard_len) {
warn!("Failed to reset SIMD decoder: {:?}, creating new one", e);
reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))?
} else {
cached_decoder
}
}
None => reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))?,
}
};
for (i, shard_opt) in shards.iter().enumerate() {
if let Some(shard) = shard_opt {
@@ -366,7 +308,7 @@ impl LegacyReedSolomonEncoder {
.write()
.map_err(|_| io::Error::other("Failed to return decoder to cache"))?;
if cache.is_none() {
*cache = Some(workspace);
*cache = Some(decoder);
}
}
@@ -391,13 +333,6 @@ impl LegacyReedSolomonEncoder {
return Ok(true);
}
let expected_bytes = shards.iter().try_fold(0usize, |total, shard| {
total
.checked_add(shard.len())
.ok_or_else(|| io::Error::from(io::ErrorKind::OutOfMemory))
})?;
let memory = current_resource_unit().and_then(|resources| resources.memory_budget());
let _expected_memory = memory.as_ref().map(|memory| memory.reserve(expected_bytes)).transpose()?;
let mut expected = shards.iter().map(|shard| Some(shard.to_vec())).collect::<Vec<_>>();
self.encode_parity(&mut expected)?;
@@ -496,18 +431,8 @@ impl ReedSolomonEncoder {
}
if let Some(ref rs) = self.encoder {
let result = if let Some(memory) = current_resource_unit().and_then(|resources| resources.memory_budget())
&& let Some(shard_len) = shards.first().map(|shard| shard.len()).filter(|len| *len != 0)
{
let scratch_len = shard_len.checked_mul(self.parity_shards).ok_or(io::ErrorKind::OutOfMemory)?;
let mut scratch = BackgroundBuffer::new(Some(memory));
scratch.ensure_len(scratch_len)?;
let mut buffers: SmallVec<[&mut [u8]; 16]> = scratch.as_mut_slice().chunks_mut(shard_len).collect();
rs.verify_with_buffer(shards, &mut buffers)
} else {
rs.verify(shards)
};
result.map_err(|e| io::Error::other(format!("Reed-Solomon verify failed: {e:?}")))
rs.verify(shards)
.map_err(|e| io::Error::other(format!("Reed-Solomon verify failed: {e:?}")))
} else {
Ok(true)
}
@@ -1275,105 +1200,6 @@ impl Erasure {
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn background_legacy_workspaces_keep_their_memory_in_cache() {
use crate::disk::{test_resource_units, with_optional_resource_unit};
use std::time::Duration;
let codec = LegacyReedSolomonEncoder::with_workspace_cache(2, 1, true).expect("isolated cached codec");
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(512, Duration::from_secs(30));
let mut shards = [vec![7; 8], vec![9; 8], vec![0; 8]];
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
codec
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
.expect("accounted encoder");
})
.await;
assert_eq!(memory.snapshot().used, 256, "fresh SIMD work storage uses at most four 64-byte units");
memory.set_sampled_limit(0, Duration::ZERO);
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
codec
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
.expect("cached payload survives shrink");
})
.await;
assert_eq!(memory.snapshot().used, 256);
memory.set_sampled_limit(512, Duration::from_secs(30));
let mut decoded = vec![None, Some(shards[1].clone()), Some(shards[2].clone())];
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
codec.reconstruct_data(&mut decoded).expect("accounted decoder");
})
.await;
assert_eq!(decoded[0].as_deref(), Some(shards[0].as_slice()));
assert_eq!(memory.snapshot().used, 512, "both cached codec payloads remain charged");
drop(codec);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn background_legacy_cache_cannot_reuse_a_different_memory_owner() {
use crate::disk::{test_resource_units, with_optional_resource_unit};
use std::time::Duration;
let codec = LegacyReedSolomonEncoder::with_workspace_cache(2, 1, true).expect("isolated cached codec");
let first = BackgroundMemoryBudget::new();
first.set_sampled_limit(256, Duration::from_secs(30));
let mut shards = [vec![7; 8], vec![9; 8], vec![0; 8]];
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(first.clone())), async {
codec
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
.expect("first owner");
})
.await;
let second = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
let error = with_optional_resource_unit(Some(resources.with_memory_budget(second.clone())), async {
codec.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
})
.await
.expect_err("a foreign cache must not bypass a closed owner");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
assert_eq!(first.snapshot().used, 0);
assert_eq!(second.snapshot().used, 0);
}
#[tokio::test]
async fn background_modern_verification_charges_scratch_and_preserves_invalid_input_errors() {
use crate::disk::{test_resource_units, with_optional_resource_unit};
use std::time::Duration;
let codec = ReedSolomonEncoder::new(2, 1).expect("modern codec");
let mut shards = [vec![7; 8], vec![9; 8], vec![0; 8]];
codec
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
.expect("valid parity");
let refs: Vec<_> = shards.iter().map(Vec::as_slice).collect();
let memory = BackgroundMemoryBudget::new();
let (resources, _) = test_resource_units(1);
let error =
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async { codec.verify(&refs) })
.await
.expect_err("closed owner rejects verification scratch");
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
memory.set_sampled_limit(8, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
assert!(codec.verify(&refs).expect("exact parity scratch fits"));
assert!(
codec.verify(&[&[], &[9; 8], &[0; 8]]).is_err(),
"empty first shard remains an error instead of a panic"
);
})
.await;
assert_eq!(memory.snapshot().used, 0);
}
use proptest::collection::{btree_set, vec};
use proptest::prelude::*;
use std::pin::Pin;
+24 -179
View File
@@ -12,10 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::background_memory::{Reservation, is_memory_pressure};
use crate::disk::disk_store::get_object_disk_read_timeout;
use crate::disk::error::{Error, Result};
use crate::disk::{BackgroundMemoryBudget, current_resource_unit};
use crate::erasure::coding::BitrotReader;
use crate::erasure::coding::BitrotWriterWrapper;
use crate::erasure::coding::encode::MultiWriter;
@@ -24,7 +22,6 @@ use futures::StreamExt;
use futures::stream::FuturesUnordered;
use std::io;
use std::io::ErrorKind;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::AsyncRead;
use tracing::{trace, warn};
@@ -33,53 +30,21 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
const EVENT_ERASURE_HEAL_STARTED: &str = "erasure_heal_started";
struct HealShards {
// Drop payloads before refunding their reservations on every error path.
shards: Vec<Option<Vec<u8>>>,
memory: Vec<Option<Reservation>>,
budget: Option<Arc<BackgroundMemoryBudget>>,
}
impl HealShards {
fn reserve_missing(&mut self, shard_size: usize) -> io::Result<()> {
if let Some(budget) = &self.budget {
for (shard, memory) in self.shards.iter().zip(&mut self.memory) {
if shard.is_none() {
*memory = Some(budget.reserve(shard_size)?);
}
}
}
Ok(())
}
fn into_bytes(self) -> Vec<Bytes> {
self.shards
.into_iter()
.zip(self.memory)
.map(|(shard, memory)| Reservation::wrap(Bytes::from(shard.unwrap_or_default()), memory))
.collect()
}
}
async fn read_heal_shards<R>(
readers: &mut [Option<BitrotReader<R>>],
shard_size: usize,
read_timeout: Duration,
) -> Result<(HealShards, Vec<Option<Error>>)>
) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>)
where
R: AsyncRead + Unpin + Send + Sync,
{
let num_readers = readers.len();
let mut shards = HealShards {
shards: vec![None; num_readers],
memory: std::iter::repeat_with(|| None).take(num_readers).collect(),
budget: current_resource_unit().and_then(|resources| resources.memory_budget()),
};
let mut shards = vec![None; num_readers];
let mut errs = vec![None; num_readers];
let mut retire_readers = Vec::new();
if shard_size == 0 {
return Ok((shards, errs));
return (shards, errs);
}
{
@@ -90,17 +55,8 @@ where
continue;
};
let budget = shards.budget.clone();
futures.push(Box::pin(async move {
let memory = match budget.as_ref().map(|budget| budget.reserve(shard_size)).transpose() {
Ok(memory) => memory,
Err(error) => return (index, Err(Error::from(error)), false),
};
let mut buf = Vec::new();
if buf.try_reserve_exact(shard_size).is_err() {
return (index, Err(Error::from(io::Error::from(ErrorKind::OutOfMemory))), false);
}
buf.resize(shard_size, 0);
let mut buf = vec![0; shard_size];
let read_result = if read_timeout.is_zero() {
reader.read(&mut buf).await
} else {
@@ -119,7 +75,7 @@ where
match read_result {
Ok(n) => {
buf.truncate(n);
(index, Ok((buf, memory)), false)
(index, Ok(buf), false)
}
Err(err) => {
let should_retire = err.kind() == ErrorKind::TimedOut;
@@ -131,14 +87,10 @@ where
while let Some((index, result, should_retire)) = futures.next().await {
match result {
Ok((shard, memory)) => {
shards.shards[index] = Some(shard);
shards.memory[index] = memory;
Ok(shard) => {
shards[index] = Some(shard);
}
Err(err) => {
if matches!(&err, Error::Io(error) if is_memory_pressure(error)) {
return Err(err);
}
errs[index] = Some(err);
if should_retire {
retire_readers.push(index);
@@ -153,7 +105,7 @@ where
warn!(shard_index = index, "retiring timed-out heal shard reader");
}
Ok((shards, errs))
(shards, errs)
}
fn heal_writer_failure(writers: &mut MultiWriter<'_>, error: io::Error) -> Error {
@@ -212,7 +164,7 @@ impl super::Erasure {
for block_index in start_block..end_block {
let shard_offset = block_index * self.shard_size();
let shard_size = self.shard_size().min(shard_file_size.saturating_sub(shard_offset));
let (mut owned_shards, errs) = read_heal_shards(&mut readers, shard_size, read_timeout).await?;
let (mut shards, errs) = read_heal_shards(&mut readers, shard_size, read_timeout).await;
// Every source shard is already bitrot-verified by its reader, so any
// data_shards survivors are sufficient to reconstruct — requiring more
@@ -231,24 +183,6 @@ impl super::Erasure {
return Err(Error::ErasureReadQuorum);
}
owned_shards.reserve_missing(shard_size)?;
let source_parity_bytes =
owned_shards
.shards
.iter()
.skip(self.data_shards)
.flatten()
.try_fold(0usize, |total, shard| {
total
.checked_add(shard.len())
.ok_or_else(|| io::Error::from(ErrorKind::OutOfMemory))
})?;
let _source_memory = owned_shards
.budget
.as_ref()
.map(|budget| budget.reserve(source_parity_bytes))
.transpose()?;
let shards = &mut owned_shards.shards;
let source_parity = shards
.iter()
.enumerate()
@@ -256,23 +190,24 @@ impl super::Erasure {
.filter_map(|(index, shard)| shard.as_ref().map(|shard| (index, shard.clone())))
.collect::<Vec<_>>();
if self.parity_shards > 0 {
self.decode_data_and_parity(shards)?;
if !source_parity.is_empty() && !self.verify_data_and_parity(shards)? {
self.decode_data_and_parity(&mut shards)?;
if !source_parity.is_empty() && !self.verify_data_and_parity(&shards)? {
return Err(Error::other("can not reconstruct data: inconsistent heal source shards"));
}
for (index, source) in &source_parity {
let Some(rebuilt) = shards[*index].as_ref() else {
for (index, source) in source_parity {
let Some(rebuilt) = shards[index].as_ref() else {
return Err(Error::other("can not reconstruct data: missing rebuilt parity shard"));
};
if rebuilt != source {
if rebuilt != &source {
return Err(Error::other("can not reconstruct data: inconsistent heal source shards"));
}
}
}
drop(source_parity);
drop(_source_memory);
let shards = owned_shards.into_bytes();
let shards = shards
.into_iter()
.map(|s| Bytes::from(s.unwrap_or_default()))
.collect::<Vec<_>>();
if let Err(error) = writers.write(shards).await {
return Err(heal_writer_failure(&mut writers, error));
@@ -371,90 +306,6 @@ mod tests {
BitrotWriterWrapper::new(CustomWriter::new_inline_buffer(), shard_size, HashAlgorithm::None)
}
#[tokio::test]
async fn heal_shards_retain_memory_through_missing_outputs_and_last_slice() {
use crate::disk::{test_resource_units, with_optional_resource_unit};
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(24, Duration::from_secs(30));
let mut readers = vec![
Some(BitrotReader::new(Cursor::new(vec![7; 8]), 8, HashAlgorithm::None, false)),
None,
Some(BitrotReader::new(Cursor::new(vec![9; 8]), 8, HashAlgorithm::None, false)),
];
let (resources, _) = test_resource_units(1);
let (mut shards, _) = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
read_heal_shards(&mut readers, 8, Duration::ZERO),
)
.await
.expect("read two accounted shards");
assert_eq!(memory.snapshot().used, 16);
shards
.reserve_missing(8)
.expect("reserve the reconstructed output before allocation");
shards.shards[1] = Some(vec![11; 8]);
assert_eq!(memory.snapshot().used, 24);
let bytes = shards.into_bytes();
let slice = bytes[1].slice(1..3);
memory.set_sampled_limit(0, Duration::ZERO);
drop(bytes);
assert_eq!(slice.as_ref(), &[11; 2]);
assert_eq!(memory.snapshot().used, 8);
drop(slice);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn heal_memory_denial_is_not_replaced_with_read_quorum() {
use crate::disk::{test_resource_units, with_optional_resource_unit};
let erasure = Erasure::new(2, 1, 16);
let encoded = erasure.encode_data(&[37; 16]).expect("valid source shards");
for limit in [0, 23, 31, 39] {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(limit, Duration::from_secs(30));
let readers = encoded
.iter()
.map(|shard| Some(BitrotReader::new(Cursor::new(shard.clone()), 8, HashAlgorithm::None, false)))
.collect();
let mut writers = vec![Some(inline_writer(8)), None, None];
let (resources, _) = test_resource_units(1);
let result = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
erasure.heal(&mut writers, readers, 16, &[]),
)
.await;
assert!(
matches!(result, Err(Error::Io(error)) if error.kind() == ErrorKind::WouldBlock),
"limit={limit}"
);
assert_eq!(memory.snapshot().used, 0);
}
}
#[tokio::test]
async fn cancelled_heal_shard_reads_release_allocated_payloads() {
use crate::disk::{test_resource_units, with_optional_resource_unit};
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(16, Duration::from_secs(30));
let mut readers = vec![
Some(BitrotReader::new(PendingReader, 8, HashAlgorithm::None, false)),
Some(BitrotReader::new(PendingReader, 8, HashAlgorithm::None, false)),
];
let (resources, _) = test_resource_units(1);
let mut read = Box::pin(with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
read_heal_shards(&mut readers, 8, Duration::ZERO),
));
assert!(futures::poll!(&mut read).is_pending());
assert_eq!(memory.snapshot().used, 16);
memory.set_sampled_limit(0, Duration::ZERO);
drop(read);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn read_heal_shards_returns_empty_slots_for_zero_sized_shards() {
let mut readers = vec![
@@ -462,11 +313,9 @@ mod tests {
None,
];
let (shards, errs) = read_heal_shards(&mut readers, 0, Duration::ZERO)
.await
.expect("empty read succeeds");
let (shards, errs) = read_heal_shards(&mut readers, 0, Duration::ZERO).await;
assert_eq!(shards.shards, vec![None, None]);
assert_eq!(shards, vec![None, None]);
assert_eq!(errs, vec![None, None]);
}
@@ -474,11 +323,9 @@ mod tests {
async fn read_heal_shards_records_reader_errors_and_retiring_timeouts() {
let mut readers = vec![Some(BitrotReader::new(FailingReader, 4, HashAlgorithm::None, false))];
let (shards, errs) = read_heal_shards(&mut readers, 4, Duration::ZERO)
.await
.expect("ordinary failures remain per-shard");
let (shards, errs) = read_heal_shards(&mut readers, 4, Duration::ZERO).await;
assert!(shards.shards[0].is_none());
assert!(shards[0].is_none());
assert!(
errs[0]
.as_ref()
@@ -487,11 +334,9 @@ mod tests {
assert!(readers[0].is_some(), "ordinary read errors must not retire readers");
let mut timeout_readers = vec![Some(BitrotReader::new(PendingReader, 4, HashAlgorithm::None, false))];
let (shards, errs) = read_heal_shards(&mut timeout_readers, 4, Duration::from_millis(1))
.await
.expect("timeouts remain per-shard");
let (shards, errs) = read_heal_shards(&mut timeout_readers, 4, Duration::from_millis(1)).await;
assert_eq!(shards.shards, vec![None]);
assert_eq!(shards, vec![None]);
assert!(errs[0].as_ref().is_some_and(|err| err.to_string().contains("timed out")));
assert!(timeout_readers[0].is_none(), "timed-out readers are retired");
}
+6 -138
View File
@@ -828,27 +828,13 @@ pub async fn create_bitrot_writer(
length: i64,
shard_size: usize,
checksum_algo: HashAlgorithm,
) -> disk::error::Result<BitrotWriterWrapper> {
create_bitrot_writer_with_orig_volume(is_inline_buffer, disk, "", volume, path, length, shard_size, checksum_algo).await
}
#[allow(clippy::too_many_arguments)]
pub async fn create_bitrot_writer_with_orig_volume(
is_inline_buffer: bool,
disk: Option<&DiskStore>,
origvolume: &str,
volume: &str,
path: &str,
length: i64,
shard_size: usize,
checksum_algo: HashAlgorithm,
) -> disk::error::Result<BitrotWriterWrapper> {
let writer = if is_inline_buffer {
CustomWriter::new_inline_buffer()
} else if let Some(disk) = disk {
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
let file = disk.create_file(origvolume, volume, path, length).await?;
let file = disk.create_file("", volume, path, length).await?;
#[cfg(feature = "hotpath")]
let file = instrument_raw_shard_writer(file, disk.is_local());
CustomWriter::new_tokio_writer(file)
@@ -1550,7 +1536,7 @@ mod tests {
writer.write(chunk).await.expect("write chunk");
}
let inline_data = writer.into_inline_data().expect("inline buffer");
let inline_data = Bytes::from(writer.into_inline_data().expect("inline buffer"));
let mut reader = create_bitrot_reader_from_bytes(
Some(inline_data),
None,
@@ -1598,7 +1584,7 @@ mod tests {
let inline_data = writer.into_inline_data().expect("inline buffer");
let mut reader = create_deferred_bitrot_reader(
Some(inline_data),
Some(inline_data.into()),
None,
"test-bucket",
"test-path",
@@ -1640,7 +1626,7 @@ mod tests {
for stripe in 0..payload.len() / shard_size {
let (mut reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
Some(inline_data.clone()),
Some(inline_data.clone().into()),
None,
"test-bucket",
"test-path",
@@ -1685,7 +1671,7 @@ mod tests {
for chunk in payload.chunks(shard_size) {
writer.write(chunk).await.expect("write chunk");
}
let mut inline_data = writer.into_inline_data().expect("inline buffer").to_vec();
let mut inline_data = writer.into_inline_data().expect("inline buffer");
// Corrupt one payload byte of block 2 (blocks are hash + data).
let block_len = algo.size() + shard_size;
@@ -1732,7 +1718,7 @@ mod tests {
let inline_data = writer.into_inline_data().expect("inline buffer");
let (mut reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
Some(inline_data),
Some(inline_data.into()),
None,
"test-bucket",
"test-path",
@@ -1859,122 +1845,4 @@ mod tests {
assert_eq!(output, data);
}
mod inline_memory_regressions {
use super::*;
use crate::disk::background_memory::Reservation;
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
use std::time::Duration;
#[tokio::test]
async fn owned_inline_reader_retains_existing_receipt_without_a_new_grant() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(64, Duration::from_secs(30));
let receipt = memory.reserve(64).expect("existing metadata payload grant");
let bytes = Reservation::wrap(Bytes::from(vec![7; 64]), Some(receipt));
memory.set_sampled_limit(0, Duration::ZERO);
let (resources, _) = test_resource_units(1);
let mut reader = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
create_bitrot_reader_from_bytes(
Some(bytes.clone()),
None,
"bucket",
"inline",
0,
64,
64,
HashAlgorithm::None,
false,
false,
),
)
.await
.expect("owned data requires no allocation")
.expect("inline reader");
drop(bytes);
assert_eq!(memory.snapshot().used, 64, "the source owner must remain attached to the reader");
let mut output = [0; 64];
assert_eq!(reader.read(&mut output).await.expect("read existing payload"), 64);
assert_eq!(output, [7; 64]);
assert_eq!(memory.snapshot().used, 64);
drop(reader);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn accounted_inline_writer_round_trips_bitrot_and_preserves_export_lifetime() {
let memory = BackgroundMemoryBudget::new();
memory.set_sampled_limit(192, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let bytes = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
let mut writer = create_bitrot_writer(true, None, "bucket", "inline", 64, 64, HashAlgorithm::HighwayHash256S)
.await
.expect("inline writer");
assert_eq!(writer.write(&[19; 64]).await.expect("complete shard"), 64);
writer.shutdown().await.expect("shutdown");
writer.into_inline_data().expect("bitrot bytes")
})
.await;
assert_eq!(bytes.len(), 96);
assert_eq!(memory.snapshot().used, 96, "exported [hash][shard] remains charged");
let (resources, _) = test_resource_units(1);
let mut reader = with_optional_resource_unit(
Some(resources.with_memory_budget(memory.clone())),
create_bitrot_reader_from_bytes(
Some(bytes.clone()),
None,
"bucket",
"inline",
0,
64,
64,
HashAlgorithm::HighwayHash256S,
false,
false,
),
)
.await
.expect("reader")
.expect("inline reader");
let mut output = [0; 64];
assert_eq!(reader.read(&mut output).await.expect("bitrot remains verified"), 64);
assert_eq!(output, [19; 64]);
assert_eq!(memory.snapshot().used, 192, "input owner and verification scratch are both live");
drop(reader);
assert_eq!(memory.snapshot().used, 96);
let slice = bytes.slice(32..36);
drop(bytes);
assert_eq!(memory.snapshot().used, 96);
drop(slice);
assert_eq!(memory.snapshot().used, 0);
}
#[tokio::test]
async fn inline_heal_writer_memory_denial_is_retryable_after_read_and_verify_fit() {
use crate::erasure::coding::Erasure;
let erasure = Erasure::new(2, 1, 16);
let encoded = erasure.encode_data(&[37; 16]).expect("foreground test fixture");
let readers = encoded
.into_iter()
.map(|shard| Some(BitrotReader::new(Cursor::new(shard), 8, HashAlgorithm::None, false)))
.collect();
let memory = BackgroundMemoryBudget::new();
// Read shards: 24 bytes; source parity: 8; verify scratch: 8.
// After verification, input shards still use 24. The target needs a
// new 40-byte [hash][shard] allocation and must fail before copying.
memory.set_sampled_limit(40, Duration::from_secs(30));
let (resources, _) = test_resource_units(1);
let result = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
let writer = create_bitrot_writer(true, None, "bucket", "inline", 8, 8, HashAlgorithm::HighwayHash256S)
.await
.expect("writer setup does not allocate payload");
let mut writers = vec![Some(writer), None, None];
erasure.heal(&mut writers, readers, 16, &[]).await
})
.await;
assert!(matches!(result, Err(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock));
assert_eq!(memory.snapshot().used, 0);
}
}
}
+1 -3
View File
@@ -55,9 +55,7 @@ mod set_disk;
mod storage_api_contracts;
mod store;
pub use disk::{
spawn_blocking_with_resource_ownership, with_background_disk_io, with_background_resource_set, with_background_resource_unit,
};
pub use disk::{with_background_disk_io, with_background_resource_unit};
pub use store::PoolMetaWriteGateStatus;
// pub mod checksum;
-24
View File
@@ -144,8 +144,6 @@ pub(crate) fn namespace_lock_signal_test_fence_is_lost(signal: &Arc<rustfs_lock:
.is_some_and(|(_, fence)| fence.is_lock_lost())
}
type FreshPutQuota = std::result::Result<(Option<crate::bucket::quota::BucketQuota>, Uuid, OffsetDateTime), String>;
#[derive(Debug)]
pub struct ObjectLockConfigSnapshot {
store_id: Option<Uuid>,
@@ -156,7 +154,6 @@ pub struct ObjectLockConfigSnapshot {
lifecycle_fence: NamespaceLockFence,
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
fresh_put_quota: std::sync::Mutex<Option<FreshPutQuota>>,
}
impl ObjectLockConfigSnapshot {
@@ -170,7 +167,6 @@ impl ObjectLockConfigSnapshot {
lifecycle_fence: NamespaceLockFence::new(),
_lifecycle_guard: None,
metadata_transaction_guard: None,
fresh_put_quota: std::sync::Mutex::new(None),
}
}
@@ -191,7 +187,6 @@ impl ObjectLockConfigSnapshot {
lifecycle_fence: NamespaceLockFence::new(),
_lifecycle_guard: None,
metadata_transaction_guard: None,
fresh_put_quota: std::sync::Mutex::new(None),
}
}
@@ -217,7 +212,6 @@ impl ObjectLockConfigSnapshot {
lifecycle_fence,
_lifecycle_guard: Some(lifecycle_guard),
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
fresh_put_quota: std::sync::Mutex::new(None),
}
}
@@ -239,23 +233,9 @@ impl ObjectLockConfigSnapshot {
lifecycle_fence,
_lifecycle_guard: None,
metadata_transaction_guard: Some(metadata_transaction_guard),
fresh_put_quota: std::sync::Mutex::new(None),
}
}
pub(crate) fn set_fresh_put_quota(&self, quota: FreshPutQuota) {
if self._lifecycle_guard.is_some() && self.metadata_transaction_guard.is_some() {
*self.fresh_put_quota.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(quota);
}
}
pub(crate) fn take_fresh_put_quota(&self) -> Option<FreshPutQuota> {
self.fresh_put_quota
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
}
#[allow(dead_code, reason = "snapshot-scope predicate asserted by this file's tests (backlog#1823)")]
pub(crate) fn is_for_store_bucket(
&self,
@@ -1543,8 +1523,6 @@ pub struct ObjectInfo {
/// published by the same upload. Callers must not repeat quota, replication,
/// lifecycle, or object-created side effects for that response.
pub multipart_completion_replayed: bool,
/// Previous current version's size observed during `rename_data` commit.
pub backfilled_old_current_size: Option<crate::disk::OldCurrentSize>,
}
impl Default for ObjectInfo {
@@ -1587,7 +1565,6 @@ impl Default for ObjectInfo {
replication_decision: String::new(),
checksum: None,
multipart_completion_replayed: false,
backfilled_old_current_size: None,
}
}
}
@@ -1631,7 +1608,6 @@ impl Clone for ObjectInfo {
replication_decision: self.replication_decision.clone(),
checksum: self.checksum.clone(),
multipart_completion_replayed: self.multipart_completion_replayed,
backfilled_old_current_size: self.backfilled_old_current_size,
expires: self.expires,
}
}
-26
View File
@@ -941,32 +941,6 @@ impl InstanceContext {
.clone()
}
/// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
///
/// Called from `ECStore::drop`. When test fixtures share the bootstrap
/// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
/// reference cycle; once the store drops the weak is dangling and any
/// `object_store()` call panics. Clearing the slot here lets the next
/// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
pub fn clear_bucket_metadata_sys_if_stale(&self) {
let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
if let Some(sys) = guard.as_ref() {
// `read()` may block briefly; the async RwLock is acceptable here
// because Drop is sync and the lock is held only long enough to
// check the weak reference.
let is_stale = {
let s = sys.try_read();
match s {
Ok(s) => s.is_store_stale(),
Err(_) => true, // contended: treat as stale-safe to clear
}
};
if is_stale {
*guard = None;
}
}
}
/// Set this instance's background-services cancellation token (once).
pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
self.background_cancel_token.set(token)
+44 -151
View File
@@ -1288,7 +1288,7 @@ async fn try_create_bitrot_readers_via_batch_pread(
// parity readers unopened; preading them here would discard their bytes
// when fill_deferred_bitrot_readers installs stripe-aligned reserves.
let data_first = strategy != BitrotReaderSetupStrategy::AllShards
&& crate::erasure::coding::decode::get_lockstep_data_shards_only_with_deferred_parity_enabled();
&& crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled();
let mut setup = BitrotReaderSetup::new(disks.len());
for (idx, disk) in disks.iter().enumerate() {
if disk.is_none() {
@@ -2549,8 +2549,6 @@ pub(in crate::set_disk) struct RenameTailCleanup {
pub(in crate::set_disk) struct RenameTailOutcome {
pub(in crate::set_disk) convergence: RenameConvergence,
pub(in crate::set_disk) cleanup: Vec<RenameTailCleanup>,
pub(in crate::set_disk) old_current_source_all_successful_checked: bool,
pub(in crate::set_disk) all_online_local_succeeded: bool,
}
const RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP";
@@ -3574,7 +3572,7 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
Guards: Send + 'static,
Finalize: FnOnce() -> FinalizeFuture + Send,
FinalizeFuture: Future<Output = ()> + Send,
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>, bool, bool) -> CleanupFuture + Send,
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>) -> CleanupFuture + Send,
CleanupFuture: Future<Output = ()> + Send,
Cleanup: 'static,
CleanupFuture: 'static,
@@ -3582,41 +3580,34 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
SubmitFuture: Future<Output = ()> + Send,
{
let tail_async_started = rustfs_io_metrics::put_stage_timer();
let (needs_heal, tail_cleanup, tail_complete, old_current_source_all_successful_checked, all_online_local_succeeded) =
match tail_drain.await {
Ok(Some(outcome)) => (
outcome.convergence.needs_heal(),
outcome.cleanup,
true,
outcome.old_current_source_all_successful_checked,
outcome.all_online_local_succeeded,
),
Ok(None) => {
warn!(
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "missing_convergence",
bucket = %request.bucket,
object = request.object_prefix.as_deref().unwrap_or_default(),
"committed rename tail drain completed without convergence"
);
(true, Vec::new(), false, false, false)
}
Err(err) => {
warn!(
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "failed",
bucket = %request.bucket,
object = request.object_prefix.as_deref().unwrap_or_default(),
error = %err,
"rename tail drain failed"
);
(true, Vec::new(), false, false, false)
}
};
let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await {
Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true),
Ok(None) => {
warn!(
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "missing_convergence",
bucket = %request.bucket,
object = request.object_prefix.as_deref().unwrap_or_default(),
"committed rename tail drain completed without convergence"
);
(true, Vec::new(), false)
}
Err(err) => {
warn!(
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
state = "failed",
bucket = %request.bucket,
object = request.object_prefix.as_deref().unwrap_or_default(),
error = %err,
"rename tail drain failed"
);
(true, Vec::new(), false)
}
};
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
tail_async_started,
@@ -3660,12 +3651,7 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
let job = RenameTailCleanupJob {
enqueued_at: Instant::now(),
targets,
future: Box::pin(cleanup(
guards,
tail_cleanup,
old_current_source_all_successful_checked,
all_online_local_succeeded,
)),
future: Box::pin(cleanup(guards, tail_cleanup)),
};
if let Err(job) = rename_tail_cleanup_queue().try_enqueue(job).await {
rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -3690,13 +3676,7 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
cleanup_submit_started,
);
let cleanup_started = rustfs_io_metrics::put_stage_timer();
cleanup(
guards,
tail_cleanup,
old_current_source_all_successful_checked,
all_online_local_succeeded,
)
.await;
cleanup(guards, tail_cleanup).await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
cleanup_started,
@@ -4175,14 +4155,7 @@ impl SetDisks {
}
}
let pending_tasks = tasks.len();
let checked_count = old_current_source_checked.iter().filter(|checked| **checked).count();
let max_source_votes = Self::max_common_old_current_source_votes(&old_current_sources);
let can_change_old_current_source = (checked_count < write_quorum
&& checked_count + pending_tasks >= write_quorum)
|| (max_source_votes < write_quorum && max_source_votes + pending_tasks >= write_quorum);
if !sent_commit && success_count >= write_quorum && !can_change_old_current_source {
if !sent_commit && success_count >= write_quorum {
let snapshot_commit = Self::rename_data_commit_from_observations(
RenameDataObservations {
disks: &coordinator_disks,
@@ -4201,8 +4174,6 @@ impl SetDisks {
cleanup_at_snapshot = commit.cleanup_disks.iter().map(Option::is_some).collect();
}
if let Some(commit_tx) = commit_tx.take() {
#[cfg(test)]
rename_fanout_barrier::wait_for_armed_rename_before_early_ack(&fanout_dst_object).await;
let _ = commit_tx.send(snapshot_commit);
}
sent_commit = true;
@@ -4325,21 +4296,9 @@ impl SetDisks {
}
}
let old_current_source_all_successful_checked = errs
.iter()
.enumerate()
.filter(|(_, err)| err.is_none())
.all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false));
let all_online_local_succeeded = coordinator_disks.iter().zip(errs.iter()).all(|(disk, err)| match disk {
Some(d) => d.is_local() && err.is_none(),
None => true,
});
Some(RenameTailOutcome {
convergence: final_convergence,
cleanup: tail_cleanup,
old_current_source_all_successful_checked,
all_online_local_succeeded,
})
}
});
@@ -4904,14 +4863,6 @@ impl SetDisks {
})
}
pub(in crate::set_disk) fn max_common_old_current_source_votes(old_current_sources: &[Option<FileInfo>]) -> usize {
let mut counts: HashMap<[u8; 32], usize> = HashMap::new();
for source in old_current_sources.iter().flatten() {
*counts.entry(Self::file_info_quorum_hash(source)).or_insert(0) += 1;
}
counts.into_values().max().unwrap_or(0)
}
pub(in crate::set_disk) fn reduce_common_old_current_source(
old_current_sources: &[Option<FileInfo>],
write_quorum: usize,
@@ -5598,11 +5549,10 @@ impl SetDisks {
write_quorum: usize,
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
) -> disk::error::Result<Vec<Option<DiskStore>>> {
self.recover_part_transaction(dst_object, write_quorum).await?;
let part = ObjectPartInfo::unmarshal(&meta)?;
let integrity = part.integrity.map(Arc::new);
self.recover_part_transaction(dst_object, write_quorum).await?;
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
@@ -5624,9 +5574,7 @@ impl SetDisks {
if let Err(error) = prepared {
return Some(Err(error));
}
if let Some(integrity) = integrity
&& !disk.is_local()
{
if let Some(integrity) = integrity {
let Some((directory, _)) = dst_object.rsplit_once('/') else {
return Some(Err(DiskError::FileCorrupt));
};
@@ -5743,36 +5691,7 @@ impl SetDisks {
return Err(err);
}
let all_prepared_succeeded = errs
.iter()
.zip(prepared_disks.iter())
.all(|(err, disk)| disk.is_none() || err.is_none());
let committed = if all_prepared_succeeded {
let settles = prepared_disks.iter().map(|disk| {
let disk = disk.clone();
let dst_object = dst_object.clone();
async move {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Commit)
.await
}
});
let settle_results = join_all(settles).await;
if settle_results.iter().filter(|res| res.is_ok()).count() >= write_quorum
&& settle_results
.iter()
.zip(prepared_disks.iter())
.all(|(res, disk)| disk.is_none() || res.is_ok())
{
true
} else {
self.recover_part_transaction(&dst_object, write_quorum).await?
}
} else {
self.recover_part_transaction(&dst_object, write_quorum).await?
};
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
if !committed {
let err = DiskError::ErasureWriteQuorum;
if let Some(context) = quorum_context {
@@ -6769,7 +6688,9 @@ impl SetDisks {
object,
);
}
if should_prevent_write(&oi, http_preconditions.if_none_match_value(), http_preconditions.if_match_value())
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
let if_match = http_preconditions.if_match_value().map(str::to_owned);
if should_prevent_write(&oi, if_none_match, if_match)
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
{
return Some(StorageError::PreconditionFailed);
@@ -7501,34 +7422,6 @@ pub(crate) mod rename_fanout_barrier {
}
}
pub(in crate::set_disk) async fn wait_for_armed_rename_before_early_ack(object: &str) {
loop {
let (any_armed, any_unpaused) = {
let reg = lock();
let mut any_armed = false;
let mut any_unpaused = false;
for ((obj, _, phase), armed) in &reg.armed {
if obj == object && *phase == Phase::Rename {
any_armed = true;
if !armed.paused.load(Ordering::SeqCst) {
any_unpaused = true;
break;
}
}
}
(any_armed, any_unpaused)
};
if !any_armed {
return;
}
if !any_unpaused {
tokio::task::yield_now().await;
return;
}
tokio::task::yield_now().await;
}
}
/// RAII scope that observes in-flight fan-out task counts for a single
/// `object`. Counts accrue only while the scope is alive; it clears its own
/// entry on drop.
@@ -8595,9 +8488,9 @@ mod tests {
}
assert_eq!(
recorder.counter_value("rustfs_io_get_object_local_batch_shards_total", &[("outcome", "success")]),
2
4
);
assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 7);
assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 14);
assert_eq!(recorder.histogram_sample_count("rustfs_io_get_object_local_batch_prepare_seconds"), 8);
}
@@ -9653,11 +9546,11 @@ mod tests {
writer.shutdown().await.expect("inline writer should shutdown");
let mut fi = base.clone();
fi.erasure.index = index + 1;
fi.data = Some(
fi.data = Some(Bytes::from(
writer
.into_inline_data()
.expect("inline bitrot writer should retain encoded data"),
);
));
files.push(fi);
}
files
@@ -11022,7 +10915,7 @@ mod tests {
move || async move {
*finalize_captured.lock().expect("finalize recorder should not poison") = true;
},
move |(), _, _, _| async move {
move |(), _| async move {
*cleanup_captured.lock().expect("cleanup recorder should not poison") = true;
},
|request| async move {
+39 -62
View File
@@ -80,7 +80,7 @@ use crate::erasure::codec::bridge::{
use crate::erasure::coding;
use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer, create_bitrot_writer_with_orig_volume};
use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_reader_from_bytes, create_bitrot_writer};
use crate::object_api::ObjectOptions;
use crate::object_api::get_object_body_cache_hook;
use crate::object_api::object_api_utils::get_raw_etag;
@@ -120,6 +120,7 @@ use bytesize::ByteSize;
use futures::future::join_all;
use http::HeaderMap;
use md5::{Digest as Md5Digest, Md5};
use regex::Regex;
use rustfs_config::MI_B;
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::{
@@ -410,6 +411,10 @@ impl ObjectLockDiagGuard {
self
}
async fn downgrade_committed_tail(&mut self) -> bool {
self.guard.downgrade_committed_tail().await
}
/// Whether the underlying namespace lock's heartbeat has observed a
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
/// point on this so a stale lock holder does not race a double-write.
@@ -4536,7 +4541,6 @@ fn resolve_tiered_decommission_write_quorum_result(
pub struct SetDisks {
pub locker_owner: String,
pub disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
pub(crate) physical_heal_admission: Arc<crate::disk::PhysicalHealBinding>,
pub set_endpoints: Vec<Endpoint>,
pub set_drive_count: usize,
pub default_parity_count: usize,
@@ -5479,7 +5483,6 @@ impl SetDisks {
Arc::new(SetDisks {
locker_owner,
disks,
physical_heal_admission: Arc::default(),
set_drive_count,
default_parity_count,
set_index,
@@ -8035,29 +8038,16 @@ fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Opti
Some(&part_numbers[index..])
}
fn canonicalized_etag(etag: &str) -> Cow<'_, str> {
// Preserve the legacy suffix-matching behavior for embedded quotes.
let trimmed = etag.trim_end_matches('"');
let Some(last_quote) = trimmed.rfind('"') else {
return Cow::Borrowed(trimmed);
};
let prefix = trimmed[..last_quote].trim_end_matches('"');
let suffix = &trimmed[last_quote + 1..];
if prefix.is_empty() {
Cow::Borrowed(suffix)
} else {
let mut result = String::with_capacity(prefix.len() + suffix.len());
result.push_str(prefix);
result.push_str(suffix);
Cow::Owned(result)
}
pub fn canonicalize_etag(etag: &str) -> String {
let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
re.replace_all(etag, "$1").to_string()
}
pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
if condition.trim() == "*" {
return true;
}
canonicalized_etag(etag) == canonicalized_etag(condition)
canonicalize_etag(etag) == canonicalize_etag(condition)
}
fn weak_etag_matches(etag: &str, condition: &str) -> bool {
@@ -8072,12 +8062,15 @@ fn weak_etag_matches(etag: &str, condition: &str) -> bool {
.or_else(|| trimmed.strip_prefix("w/"))
.unwrap_or(trimmed)
}
canonicalized_etag(strip_weak(etag)) == canonicalized_etag(strip_weak(condition))
canonicalize_etag(strip_weak(etag)) == canonicalize_etag(strip_weak(condition))
}
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<&str>, if_match: Option<&str>) -> bool {
let if_none_match = if_none_match.map(str::trim).filter(|condition| !condition.is_empty());
let if_match = if_match.map(str::trim).filter(|condition| !condition.is_empty());
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
let if_none_match = if_none_match
.as_deref()
.map(str::trim)
.filter(|condition| !condition.is_empty());
let if_match = if_match.as_deref().map(str::trim).filter(|condition| !condition.is_empty());
match &oi.etag {
Some(etag) => {
@@ -13781,22 +13774,6 @@ mod tests {
assert!(result2.iter().all(|d| d.is_none()));
}
#[test]
fn canonicalized_etag_preserves_legacy_regex_semantics() {
let regex = regex::Regex::new("\"*?([^\"]*?)\"*?$").expect("valid legacy ETag pattern");
let alphabet = ['a', '"', '\n', 'é', '\0'];
for length in 0..=6_u32 {
for mut number in 0..alphabet.len().pow(length) {
let mut input = String::new();
for _ in 0..length {
input.push(alphabet[number % alphabet.len()]);
number /= alphabet.len();
}
assert_eq!(canonicalized_etag(&input), regex.replace_all(&input, "$1"), "input: {input:?}");
}
}
}
#[test]
fn test_etag_matches() {
assert!(e_tag_matches("abc", "abc"));
@@ -14372,61 +14349,61 @@ mod tests {
etag: Some("abc".to_string()),
..Default::default()
};
let if_none_match = Some("abc");
let if_none_match = Some("abc".to_string());
let if_match = None;
assert!(should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = Some("*");
let if_none_match = Some("*".to_string());
let if_match = None;
assert!(should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = None;
let if_match = Some("def");
let if_match = Some("def".to_string());
assert!(should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = None;
let if_match = Some("*");
let if_match = Some("*".to_string());
assert!(!should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = Some("def");
let if_none_match = Some("def".to_string());
let if_match = None;
assert!(!should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = Some("def");
let if_match = Some("*");
let if_none_match = Some("def".to_string());
let if_match = Some("*".to_string());
assert!(!should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = Some("def");
let if_match = Some("\"abc\"");
let if_none_match = Some("def".to_string());
let if_match = Some("\"abc\"".to_string());
assert!(!should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = Some("*");
let if_match = Some("\"abc\"");
let if_none_match = Some("*".to_string());
let if_match = Some("\"abc\"".to_string());
assert!(should_prevent_write(&oi, if_none_match, if_match));
let oi = ObjectInfo {
etag: None,
..Default::default()
};
let if_none_match = Some("*");
let if_match = Some("\"abc\"");
let if_none_match = Some("*".to_string());
let if_match = Some("\"abc\"".to_string());
assert!(should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = None;
let if_match = None;
assert!(!should_prevent_write(&oi, if_none_match, if_match));
let if_none_match = Some("");
let if_match = Some(" ");
let if_none_match = Some(String::new());
let if_match = Some(" ".to_string());
assert!(!should_prevent_write(&oi, if_none_match, if_match));
let oi = ObjectInfo {
etag: Some("abc".to_string()),
..Default::default()
};
assert!(should_prevent_write(&oi, Some("W/\"abc\""), None));
assert!(!should_prevent_write(&oi, Some("W/\"def\""), None));
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"")));
assert!(should_prevent_write(&oi, Some("W/\"abc\"".to_string()), None));
assert!(!should_prevent_write(&oi, Some("W/\"def\"".to_string()), None));
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"".to_string())));
}
#[test]
@@ -14834,7 +14811,7 @@ mod tests {
file.erasure.index = files.len() + 1;
file.erasure.block_size = erasure.block_size;
file.uses_legacy_checksum = uses_legacy;
file.data = Some(data);
file.data = Some(Bytes::from(data));
files.push(file);
}
@@ -15295,7 +15272,7 @@ mod tests {
}
let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
disk.write_all(bucket, &shard_path, encoded)
disk.write_all(bucket, &shard_path, Bytes::from(encoded))
.await
.expect("encoded shard should be stored");
@@ -15379,7 +15356,7 @@ mod tests {
}
let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
disk.write_all(bucket, &shard_path, encoded)
disk.write_all(bucket, &shard_path, Bytes::from(encoded))
.await
.expect("encoded shard should be stored");
@@ -15492,7 +15469,7 @@ mod tests {
}
let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
disk.write_all(bucket, &shard_path, encoded)
disk.write_all(bucket, &shard_path, Bytes::from(encoded))
.await
.expect("encoded shard should be stored");
}
@@ -97,14 +97,6 @@ pub(crate) mod test_hooks {
.push((bucket.to_owned(), object.to_owned(), hook));
}
pub(super) fn has_before_gc_finalize(bucket: &str, object: &str) -> bool {
BEFORE_GC_FINALIZE
.lock()
.expect("GC finalize hooks")
.iter()
.any(|(hook_bucket, hook_object, _)| hook_bucket == bucket && hook_object == object)
}
pub(super) async fn run_before_gc_finalize(bucket: &str, object: &str) {
let hook = {
let mut hooks = BEFORE_GC_FINALIZE.lock().expect("GC finalize hooks");
@@ -426,59 +418,6 @@ impl SetDisks {
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
let can_commit = context.can_commit;
let can_try_local_fast_path = disks.len() == self.set_drive_count
&& disks.iter().all(|d| d.as_ref().is_some_and(|d| d.is_local()))
&& !crate::disk::local::has_delete_version_test_failpoints()
&& {
#[cfg(test)]
{
!test_hooks::has_before_gc_finalize(bucket, object)
}
#[cfg(not(test))]
{
true
}
};
if can_try_local_fast_path && can_commit() {
let fast_published = join_all(disks.iter().map(|disk| async {
if !can_commit() {
return Err(DiskError::other("delete recovery namespace fence was lost"));
}
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
.await
}))
.await;
let all_fast_published = fast_published.len() == self.set_drive_count
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
if all_fast_published && can_commit() {
#[cfg(test)]
test_hooks::record_cleanup_attempt(bucket, object);
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
let ns_owner = context.namespace_owner.clone();
async move {
if !can_commit() {
return Ok(false);
}
let Some(disk) = disk.as_ref() else {
return Ok(false);
};
let Ok(Some((journal, _, _))) = pub_res else {
return Ok(false);
};
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
.await
}
}))
.await;
if finalized.iter().all(|res| matches!(res, Ok(true))) {
if let Some(ticket) = context.recovery_ticket.as_ref() {
self.ctx.delete_recovery_hints().ack(ticket);
}
return Ok(());
}
}
}
let states = inspect(disks, bucket, object, transaction).await;
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
+25 -634
View File
@@ -13,14 +13,14 @@
// limitations under the License.
use super::super::{
CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, DeleteOptions, DiskError, DiskStore, DriveState, EVENT_SET_DISK_HEAL, Error,
FileInfo, HashAlgorithm, HashMap, HealDriveInfo, HealItemType, HealOpts, HealResultItem, HealScanMode, Infos,
Bytes, CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, DeleteOptions, DiskError, DiskStore, DriveState, EVENT_SET_DISK_HEAL,
Error, FileInfo, HashAlgorithm, HashMap, HealDriveInfo, HealItemType, HealOpts, HealResultItem, HealScanMode, Infos,
LOG_SUBSYSTEM_SET_DISK, ObjectInfo, ObjectOptions, ObjectPartInfo, Path, RUSTFS_META_TMP_BUCKET, ReadOptions, Result,
SLASH_SEPARATOR, SetDisks, StorageError, Uuid, coding, count_errs, count_part_not_success, create_bitrot_writer, debug, disk,
disks_with_all_parts, encode_dir_object, error, file_info_is_valid_for_metadata, formats_match_reference_slots,
get_format_erasure_in_quorum, get_lock_acquire_timeout, has_suffix, heal_bucket_local_on_disks, is_object_dir_dangling,
join_all, load_format_erasure_all, path_join_buf, save_format_file, should_heal_object_on_disk, stat_all_dirs, to_object_err,
warn,
SLASH_SEPARATOR, SetDisks, StorageError, Uuid, coding, count_errs, count_part_not_success, create_bitrot_reader,
create_bitrot_writer, debug, disk, disks_with_all_parts, encode_dir_object, error, file_info_is_valid_for_metadata,
formats_match_reference_slots, get_format_erasure_in_quorum, get_lock_acquire_timeout, has_suffix,
heal_bucket_local_on_disks, is_object_dir_dangling, join_all, load_format_erasure_all, path_join_buf, save_format_file,
should_heal_object_on_disk, stat_all_dirs, to_object_err, warn,
};
use crate::bucket::retirement::MarkerRetirementContext;
use crate::diagnostics::object_lock::ObjectLockAttempt;
@@ -52,7 +52,7 @@ fn pin_object_heal_future<F: std::future::Future>(build: impl FnOnce() -> F) ->
use crate::disk::DataDirDeleteStatus;
use crate::disk::DiskAPI;
use crate::disk::local::{DELETE_DATA_DIR_MARKER_PREFIX, metadata_less_part_file};
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, object_mmap_read_enabled};
use crate::io_support::bitrot::object_mmap_read_enabled;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use rustfs_common::mrf_channel::MrfDeleteMarkerPurge;
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
@@ -1236,7 +1236,6 @@ impl SetDisks {
result.object_size =
ObjectInfo::from_file_info(&latest_meta, bucket, object, true).get_actual_size()? as usize;
result.resolved_version_id = Some(*latest_meta.version_id.unwrap_or_default().as_bytes());
result.quorum_selected_data_dir = selected_heal_data_dir(&latest_meta, opts);
// Loop to find number of disks with valid data, per-drive
// data state and a list of outdated disks on which data needs
// to be healed.
@@ -1424,9 +1423,13 @@ impl SetDisks {
);
}
// A no-remove request cannot turn surviving data into dangling
// cleanup or its grace deferral.
if read_repair_uses_shared_lock || preserve_unrepairable || !opts.remove {
// A no-remove request cannot turn surviving same-version
// data into dangling cleanup or its grace deferral.
let preserve_surviving_data = !opts.remove
&& data_errs_by_disk
.values()
.any(|parts| !parts.is_empty() && parts.iter().all(|status| *status == CHECK_PART_SUCCESS));
if read_repair_uses_shared_lock || preserve_unrepairable || preserve_surviving_data {
return Ok((result, Some(cannot_heal_err)));
}
@@ -1812,8 +1815,8 @@ impl SetDisks {
checksum_info.algorithm
};
match create_bitrot_reader_from_bytes(
metadata.data.clone(),
match create_bitrot_reader(
metadata.data.as_deref(),
Some(disk),
bucket,
&path_join_buf(&[object, &src_data_dir, &format!("part.{}", part.number)]),
@@ -1910,7 +1913,8 @@ impl SetDisks {
// if let Some(w) = writer.as_any().downcast_ref::<BitrotFileWriter>() {
// parts_metadata[index].data = Some(w.inline_data().to_vec());
// }
parts_metadata[index].data = Some(writer.into_inline_data().unwrap_or_default());
parts_metadata[index].data =
Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default());
}
parts_metadata[index].set_inline_data();
} else {
@@ -2198,13 +2202,6 @@ impl SetDisks {
}
}
Err(err) => {
// All replicas explicitly missing was handled before the
// quorum check. A partial read cannot prove object absence.
let err = if matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound) {
DiskError::ErasureReadQuorum
} else {
err
};
if let Some(retirement) = retirement
&& parts_metadata
.iter()
@@ -2244,8 +2241,7 @@ impl SetDisks {
}
return Ok((item, cleanup.err()));
}
if !opts.dry_run
&& allow_explicit_version_regen
if allow_explicit_version_regen
&& !version_id.is_empty()
&& self
.try_regenerate_explicit_version_meta(bucket, object, version_id, &parts_metadata, &errs, &disks)
@@ -2264,7 +2260,7 @@ impl SetDisks {
.await;
}
if opts.dry_run || preserve_unrepairable || !opts.remove {
if opts.dry_run || preserve_unrepairable {
return Ok((
self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
.await,
@@ -3265,13 +3261,6 @@ impl SetDisks {
}
}
fn selected_heal_data_dir(metadata: &FileInfo, opts: &HealOpts) -> Option<[u8; 16]> {
if opts.read_repair || opts.dry_run || metadata.deleted || metadata.is_remote() {
return None;
}
metadata.data_dir.filter(|id| !id.is_nil()).map(|id| *id.as_bytes())
}
fn finalize_object_heal_result(
mut result: HealResultItem,
mut error: Option<Error>,
@@ -3295,9 +3284,6 @@ fn finalize_object_heal_result(
achieved: 0,
});
}
if error.is_some() {
result.quorum_selected_data_dir = None;
}
(result, error)
}
@@ -4012,7 +3998,6 @@ mod heal_result_report_tests {
metadata_verified: true,
metadata_repair_verified: true,
legacy_identity_unverified: true,
quorum_selected_data_dir: Some([1; 16]),
..Default::default()
};
@@ -4023,7 +4008,6 @@ mod heal_result_report_tests {
assert!(!result.metadata_verified);
assert!(!result.metadata_repair_verified);
assert!(!result.legacy_identity_unverified);
assert_eq!(result.quorum_selected_data_dir, None);
assert!(absence.is_none());
assert!(matches!(
error,
@@ -4037,134 +4021,6 @@ mod heal_result_report_tests {
));
}
#[test]
fn selected_data_dir_is_a_quorum_metadata_observation() {
let generation = Uuid::new_v4();
let mut metadata = FileInfo {
data_dir: Some(generation),
..Default::default()
};
let opts = HealOpts::default();
assert_eq!(super::selected_heal_data_dir(&metadata, &opts), Some(*generation.as_bytes()));
assert_eq!(
super::selected_heal_data_dir(&metadata, &HealOpts { no_lock: true, ..opts }),
Some(*generation.as_bytes())
);
assert_eq!(super::selected_heal_data_dir(&metadata, &HealOpts { dry_run: true, ..opts }), None);
assert_eq!(
super::selected_heal_data_dir(
&metadata,
&HealOpts {
read_repair: true,
..opts
}
),
None
);
metadata.data_dir = Some(Uuid::nil());
assert_eq!(super::selected_heal_data_dir(&metadata, &opts), None);
metadata.data_dir = None;
assert_eq!(super::selected_heal_data_dir(&metadata, &opts), None);
metadata.data_dir = Some(generation);
metadata.deleted = true;
assert_eq!(super::selected_heal_data_dir(&metadata, &opts), None);
metadata.deleted = false;
metadata.transition_status = TRANSITION_COMPLETE.to_owned();
assert_eq!(super::selected_heal_data_dir(&metadata, &opts), None);
}
#[test]
fn failed_heal_does_not_publish_selected_generation() {
let result = rustfs_madmin::heal_commands::HealResultItem {
quorum_selected_data_dir: Some([1; 16]),
..Default::default()
};
let mut absence = None;
let (result, error) =
super::finalize_object_heal_result(result, Some(Error::FileNotFound), &mut absence, false, "b", "o");
assert!(error.is_some());
assert_eq!(result.quorum_selected_data_dir, None);
}
#[tokio::test]
async fn heal_result_observes_current_generation_after_unversioned_overwrite() {
let (_temp_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = "heal-result-generation";
let object = "object.bin";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create test bucket");
let put_opts = ObjectOptions {
no_lock: true,
..Default::default()
};
let mut first = PutObjReader::from_vec(vec![0x11; 1024 * 1024]);
let first_info = set
.put_object(bucket, object, &mut first, &put_opts)
.await
.expect("write first generation");
let first_generation = first_info.data_dir.expect("first object data directory");
let heal_opts = HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
};
let (first_result, first_error) =
crate::storage_api_contracts::heal::HealOperations::heal_object(set.as_ref(), bucket, object, "", &heal_opts)
.await
.expect("heal first generation");
assert!(first_error.is_none());
assert_eq!(first_result.quorum_selected_data_dir, Some(*first_generation.as_bytes()));
let mut second = PutObjReader::from_vec(vec![0x22; 1024 * 1024]);
let second_info = set
.put_object(bucket, object, &mut second, &put_opts)
.await
.expect("overwrite unversioned object");
let second_generation = second_info.data_dir.expect("second object data directory");
assert_ne!(first_generation, second_generation);
let (second_result, second_error) =
crate::storage_api_contracts::heal::HealOperations::heal_object(set.as_ref(), bucket, object, "", &heal_opts)
.await
.expect("heal second generation");
assert!(second_error.is_none());
assert_eq!(second_result.quorum_selected_data_dir, Some(*second_generation.as_bytes()));
assert_ne!(second_result.quorum_selected_data_dir, first_result.quorum_selected_data_dir);
let (unlocked_result, unlocked_error) = crate::storage_api_contracts::heal::HealOperations::heal_object(
set.as_ref(),
bucket,
object,
"",
&HealOpts {
no_lock: true,
..heal_opts
},
)
.await
.expect("observe quorum selection without a namespace lock");
assert!(unlocked_error.is_none());
assert_eq!(unlocked_result.quorum_selected_data_dir, Some(*second_generation.as_bytes()));
for opts in [
HealOpts {
read_repair: true,
..heal_opts
},
HealOpts {
dry_run: true,
..heal_opts
},
] {
let (result, error) =
crate::storage_api_contracts::heal::HealOperations::heal_object(set.as_ref(), bucket, object, "", &opts)
.await
.expect("observe heal without an eligible lock");
assert!(error.is_none());
assert_eq!(result.quorum_selected_data_dir, None);
}
}
#[test]
fn absent_and_nil_selected_versions_are_the_same_null_identity() {
let requested_nil = Uuid::nil().to_string();
@@ -7669,13 +7525,12 @@ mod heal_result_report_tests {
"",
&HealOpts {
no_lock: true,
remove: true,
..Default::default()
},
)
.await
.expect("unsafe dangling state should be reported without deletion");
assert_eq!(err, Some(DiskError::ErasureReadQuorum));
assert_eq!(err, Some(DiskError::FileNotFound));
let surviving = disks[0]
.as_ref()
.expect("first test disk should be online")
@@ -7716,7 +7571,6 @@ mod heal_result_report_tests {
.expect("seed exact stale historical version");
let opts = HealOpts {
no_lock: true,
remove: true,
scan_mode: HealScanMode::Deep,
..Default::default()
};
@@ -7792,443 +7646,6 @@ mod heal_result_report_tests {
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn heal_without_remove_preserves_unrecoverable_unversioned_shards() {
use crate::disk::{CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS};
use crate::object_api::WriteCompletion;
use futures::FutureExt as _;
use std::panic::AssertUnwindSafe;
use std::time::Duration;
// Exercise the same on-disk layout with explicit removal first, so
// an unrelated early failure cannot masquerade as preservation.
for remove in [true, false] {
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let outcome = AssertUnwindSafe(tokio::time::timeout(Duration::from_secs(45), async {
let bucket = if remove {
"heal-remove-unversioned"
} else {
"heal-preserve-unversioned"
};
let object = "object.bin";
let old_time = OffsetDateTime::from_unix_timestamp(1).expect("fixture modification time");
let elapsed = (OffsetDateTime::now_utc() - old_time).whole_seconds();
let grace = rustfs_utils::get_env_u64("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", 3600);
assert!(
elapsed > i64::try_from(grace).unwrap_or(i64::MAX),
"fixture must be outside the configured dangling-delete grace period"
);
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create fixture bucket");
let options = ObjectOptions {
mod_time: Some(old_time),
write_completion: WriteCompletion::TailDrained,
..Default::default()
};
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x5a; 1024 * 1024]), &options)
.await
.expect("write all four fixture shards");
let mut metadata = Vec::new();
let mut infos = Vec::new();
for (dir, disk) in dirs.iter().zip(&disks) {
metadata.push(
tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE))
.await
.expect("read each committed metadata file"),
);
infos.push(
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("read each committed unversioned metadata replica"),
);
}
let selected = &infos[0];
let data_dir = selected.data_dir.expect("fixture must be non-inline");
assert!(selected.version_id.is_none_or(|id| id.is_nil()));
assert_eq!(selected.mod_time, Some(old_time));
let missing_count = selected.erasure.parity_blocks + 1;
assert!(missing_count < disks.len(), "at least one physical shard must survive");
assert!(infos.iter().all(|info| {
info.data_dir == Some(data_dir)
&& info.version_id.is_none_or(|id| id.is_nil())
&& info.erasure.parity_blocks == selected.erasure.parity_blocks
}));
let part_path = |dir: &TempDir| dir.path().join(bucket).join(object).join(data_dir.to_string()).join("part.1");
let surviving_shard = tokio::fs::read(part_path(&dirs[missing_count]))
.await
.expect("read the shard that must survive");
for dir in dirs.iter().take(missing_count) {
tokio::fs::remove_file(part_path(dir)).await.expect("remove one exact shard");
}
for (index, (disk, info)) in disks.iter().zip(&infos).enumerate() {
let checked = disk.check_parts(bucket, object, info).await.expect("check fixture shards");
let expected = if index < missing_count {
CHECK_PART_FILE_NOT_FOUND
} else {
CHECK_PART_SUCCESS
};
assert_eq!(checked.results, vec![expected], "disk {index} has the intended shard state");
}
let visible = set
.get_object_info(
bucket,
object,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("metadata quorum remains visible before heal");
assert_eq!(visible.data_dir, Some(data_dir));
let (result, error) = <SetDisks as crate::storage_api_contracts::heal::HealOperations>::heal_object(
set.as_ref(),
bucket,
object,
"",
&HealOpts {
remove,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("heal returns an object result");
if remove {
assert!(error.is_none(), "explicit dangling cleanup must succeed: {error:?}");
for dir in &dirs {
assert!(
tokio::fs::metadata(dir.path().join(bucket))
.await
.expect("bucket remains readable")
.is_dir()
);
assert!(
matches!(
tokio::fs::symlink_metadata(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)).await,
Err(error) if error.kind() == std::io::ErrorKind::NotFound
),
"explicit removal must remove the metadata replica"
);
}
assert!(
matches!(tokio::fs::symlink_metadata(part_path(&dirs[missing_count])).await,
Err(error) if error.kind() == std::io::ErrorKind::NotFound),
"explicit removal must remove the surviving shard"
);
assert_eq!(result.after.drives.len(), disks.len());
assert!(
result
.after
.drives
.iter()
.all(|drive| drive.state == DriveState::Missing.to_string())
);
} else {
for (index, (dir, before)) in dirs.iter().zip(&metadata).enumerate() {
let after = tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE))
.await
.expect("remove=false must keep each metadata replica");
assert_eq!(&after, before, "remove=false changed metadata on disk {index}");
}
assert_eq!(
tokio::fs::read(part_path(&dirs[missing_count]))
.await
.expect("remove=false must keep the surviving shard"),
surviving_shard
);
assert!(matches!(&error, Some(Error::ErasureReadQuorum)), "unexpected heal result: {error:?}");
assert_eq!(result.before.drives.len(), disks.len());
assert_eq!(
result
.before
.drives
.iter()
.filter(|drive| drive.state == DriveState::Missing.to_string())
.count(),
missing_count
);
let mut absence = None;
let (_, proof_error) = set
.heal_object_with_absence(
bucket,
object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: true,
},
)
.await
.expect("repeat through the absence-proof path");
assert!(matches!(proof_error, Some(Error::ErasureReadQuorum)));
assert!(absence.is_none(), "surviving shards cannot produce an absence proof");
}
}))
.catch_unwind()
.await;
drop(set);
drop(disks);
if !matches!(&outcome, Ok(Ok(()))) {
let retained: Vec<_> = dirs.into_iter().map(TempDir::keep).collect();
eprintln!("unrecoverable-shard fixture retained: {retained:?}");
match outcome {
Ok(Err(_)) => panic!("unrecoverable-shard fixture timed out"),
Err(panic) => std::panic::resume_unwind(panic),
Ok(Ok(())) => unreachable!(),
}
}
}
}
#[tokio::test]
#[serial_test::serial]
async fn heal_without_remove_preserves_explicit_version_metadata_during_dry_run() {
use crate::object_api::{ShardIntegrityWriteMode, WriteCompletion};
use futures::FutureExt as _;
use std::panic::AssertUnwindSafe;
use std::time::Duration;
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let outcome = AssertUnwindSafe(tokio::time::timeout(Duration::from_secs(45), async {
let bucket = "heal-dry-run-versioned";
let object = "object.bin";
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create fixture bucket");
let original = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![0x67; 1024 * 1024]),
&ObjectOptions {
versioned: true,
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("write a real version on all four disks");
let version = original.version_id.expect("explicit version");
assert!(!version.is_nil());
let generation = original.data_dir.expect("external data generation");
let version_string = version.to_string();
let metadata_path = |dir: &TempDir| dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
let shard_path = |dir: &TempDir| {
dir.path()
.join(bucket)
.join(object)
.join(generation.to_string())
.join("part.1")
};
let mut shards = Vec::new();
for (dir, disk) in dirs.iter().zip(&disks) {
let info = disk
.read_version("", bucket, object, &version_string, &ReadOptions::default())
.await
.expect("initial metadata on each disk");
assert_eq!(info.version_id, Some(version));
assert_eq!(info.data_dir, Some(generation));
assert!(info.parts.iter().all(|part| part.integrity.is_none()));
assert!(
rustfs_filemeta::shard_integrity::descriptor_from_metadata(&info.metadata)
.expect("metadata protection descriptor")
.is_none()
);
shards.push(tokio::fs::read(shard_path(dir)).await.expect("save original shard"));
}
let surviving_metadata = tokio::fs::read(metadata_path(&dirs[0]))
.await
.expect("save surviving metadata");
for dir in dirs.iter().skip(1) {
tokio::fs::remove_file(metadata_path(dir))
.await
.expect("remove one metadata replica");
}
for disk in disks.iter().skip(1) {
assert!(matches!(
disk.read_version("", bucket, object, &version_string, &ReadOptions::default())
.await,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
));
}
let mut absence = None;
let (_, error) = set
.heal_object_with_absence(
bucket,
object,
&version_string,
&HealOpts {
dry_run: true,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
.expect("dry-run reports the metadata quorum failure");
assert!(matches!(error, Some(Error::ErasureReadQuorum)), "partial metadata must not look absent");
assert!(absence.is_none());
assert_eq!(
tokio::fs::read(metadata_path(&dirs[0])).await.expect("surviving metadata"),
surviving_metadata
);
for (index, dir) in dirs.iter().enumerate() {
if index != 0 {
assert!(matches!(
tokio::fs::symlink_metadata(metadata_path(dir)).await,
Err(error) if error.kind() == std::io::ErrorKind::NotFound
));
}
assert_eq!(tokio::fs::read(shard_path(dir)).await.expect("original shard"), shards[index]);
}
let (_, error) = set
.heal_object_with_absence(
bucket,
object,
&version_string,
&HealOpts::default(),
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: false,
},
)
.await
.expect("normal heal attempts metadata regeneration");
assert!(error.is_none(), "normal heal must repair the same layout: {error:?}");
assert!(absence.is_none());
for (index, (dir, disk)) in dirs.iter().zip(&disks).enumerate() {
let info = disk
.read_version("", bucket, object, &version_string, &ReadOptions::default())
.await
.expect("normal heal restored metadata");
assert_eq!(info.version_id, Some(version));
assert_eq!(info.data_dir, Some(generation));
assert_eq!(tokio::fs::read(shard_path(dir)).await.expect("original shard"), shards[index]);
}
}))
.catch_unwind()
.await;
drop(set);
drop(disks);
if !matches!(&outcome, Ok(Ok(()))) {
let retained: Vec<_> = dirs.into_iter().map(TempDir::keep).collect();
eprintln!("dry-run versioned fixture retained: {retained:?}");
match outcome {
Ok(Err(_)) => panic!("dry-run versioned fixture timed out"),
Err(panic) => std::panic::resume_unwind(panic),
Ok(Ok(())) => unreachable!(),
}
}
}
#[tokio::test]
#[serial_test::serial]
async fn heal_without_remove_preserves_dangling_inline_metadata_and_emits_no_absence_proof() {
use futures::FutureExt as _;
use std::panic::AssertUnwindSafe;
use std::time::Duration;
temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async {
for remove in [true, false] {
let bucket = if remove {
"inline-dangling-remove"
} else {
"inline-dangling-preserve"
};
let object = "stale.txt";
let (dirs, set, disks) =
dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await;
let outcome = AssertUnwindSafe(tokio::time::timeout(Duration::from_secs(45), async {
let metadata = dirs[0].path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
let before = tokio::fs::read(&metadata).await.expect("read surviving inline metadata");
let mut absence = None;
let (_, error) = set
.heal_object_with_absence(
bucket,
object,
"",
&HealOpts {
no_lock: true,
remove,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: true,
},
)
.await
.expect("dangling inline heal returns an object result");
if remove {
assert!(error.is_none(), "explicit cleanup must succeed: {error:?}");
assert!(
absence.as_ref().is_some_and(|proof| proof.removed),
"successful cleanup must prove removal"
);
assert!(matches!(
tokio::fs::symlink_metadata(metadata).await,
Err(error) if error.kind() == std::io::ErrorKind::NotFound
));
let (_, missing_error) = set
.heal_object_with_absence(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: true,
},
)
.await
.expect("fully absent object returns a result");
assert!(matches!(missing_error, Some(Error::FileNotFound)));
assert!(absence.is_some_and(|proof| !proof.removed));
} else {
assert!(matches!(error, Some(Error::ErasureReadQuorum)), "partial metadata must not look absent");
assert!(absence.is_none(), "a surviving candidate cannot prove authoritative absence");
assert_eq!(tokio::fs::read(metadata).await.expect("metadata must remain"), before);
}
}))
.catch_unwind()
.await;
drop(set);
drop(disks);
if !matches!(&outcome, Ok(Ok(()))) {
let retained: Vec<_> = dirs.into_iter().map(TempDir::keep).collect();
eprintln!("dangling-inline fixture retained: {retained:?}");
match outcome {
Ok(Err(_)) => panic!("dangling-inline fixture timed out"),
Err(panic) => std::panic::resume_unwind(panic),
Ok(Ok(())) => unreachable!(),
}
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn heal_reports_success_after_dangling_inline_cleanup() {
@@ -8245,7 +7662,6 @@ mod heal_result_report_tests {
"",
&HealOpts {
no_lock: true,
remove: true,
..Default::default()
},
)
@@ -8276,7 +7692,6 @@ mod heal_result_report_tests {
"",
&HealOpts {
no_lock: true,
remove: true,
..Default::default()
},
)
@@ -8327,14 +7742,13 @@ mod heal_result_report_tests {
&HealOpts {
dry_run: true,
no_lock: true,
remove: true,
..Default::default()
},
)
.await
.expect("dry-run should report the dangling candidate without mutation");
assert_eq!(error, Some(DiskError::ErasureReadQuorum));
assert_eq!(error, Some(DiskError::FileNotFound));
assert!(
temp_dirs[0]
.path()
@@ -8358,27 +7772,6 @@ mod heal_result_report_tests {
seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
set.disks.write().await[1] = None;
let mut absence = None;
let (_, error) = set
.heal_object_with_absence(
bucket,
object,
"",
&HealOpts {
no_lock: true,
..Default::default()
},
&mut absence,
AbsenceProofRequest {
retirement: None,
allow_unversioned: true,
},
)
.await
.expect("offline shard state should be reported without removal");
assert!(matches!(error, Some(Error::ErasureReadQuorum)));
assert!(absence.is_none(), "offline members cannot prove absence");
let (_, err) = set
.heal_object(
bucket,
@@ -8386,13 +7779,12 @@ mod heal_result_report_tests {
"",
&HealOpts {
no_lock: true,
remove: true,
..Default::default()
},
)
.await
.expect("offline shard state should be reported without deletion");
assert_eq!(err, Some(DiskError::ErasureReadQuorum));
assert_eq!(err, Some(DiskError::FileNotFound));
let surviving = disks[0]
.as_ref()
.expect("first test disk should be online")
@@ -8429,13 +7821,12 @@ mod heal_result_report_tests {
"",
&HealOpts {
no_lock: true,
remove: true,
..Default::default()
},
)
.await
.expect("part probe timeout should be reported without deletion");
assert_eq!(err, Some(DiskError::ErasureReadQuorum));
assert_eq!(err, Some(DiskError::FileNotFound));
let surviving = disks[0]
.as_ref()
.expect("first test disk should be online")
+5 -8
View File
@@ -3471,6 +3471,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
bind_multipart_proof_snapshot(&disks, &self.multipart_proof_verification_context())?;
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
let part_meta_paths = uploaded_parts
.iter()
.map(|v| format!("{part_path}part.{0}.meta", v.part_num))
@@ -3478,10 +3482,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
let object_parts = Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
@@ -4389,7 +4389,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let needs_immediate_heal = rename_commit.needs_immediate_heal();
let op_old_dir = rename_commit.data_dir;
let cleanup_disks = rename_commit.cleanup_disks;
let old_current_size = rename_commit.old_current_size;
let committed_file_info = rename_commit.committed_file_info;
if needs_immediate_heal {
@@ -4504,9 +4503,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
drop(_upload_guard.take());
let mut obj_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned);
obj_info.backfilled_old_current_size = old_current_size;
Ok(obj_info)
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
};
let result = if detach_commit_owner {
+77 -37
View File
@@ -46,9 +46,9 @@ use super::super::{
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer_with_orig_volume, debug,
delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, delete_file_info_version_id,
disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
@@ -158,15 +158,13 @@ fn rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() -> bool {
}
fn tier_free_version_source_reuse_rename_data_enabled() -> bool {
#[cfg(test)]
{
rustfs_utils::get_env_bool(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV, true)
}
#[cfg(not(test))]
{
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ENABLED.get_or_init(|| rustfs_utils::get_env_bool(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV, true))
}
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ENABLED.get_or_init(|| {
matches!(
std::env::var(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
)
})
}
fn tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled() -> bool {
@@ -4459,10 +4457,9 @@ impl SetDisks {
if let Some(disk) = disk_op
&& disk.is_online().await
{
match create_bitrot_writer_with_orig_volume(
match create_bitrot_writer(
is_inline_buffer,
Some(disk),
bucket,
RUSTFS_META_TMP_BUCKET,
&tmp_obj,
shard_file_size,
@@ -4744,7 +4741,7 @@ impl SetDisks {
.ok_or_else(|| Error::other(format!("inline encoder omitted disk shard {i}")))?,
);
} else if let Some(writer) = writers[i].take() {
pfi.data = Some(writer.into_inline_data().unwrap_or_default());
pfi.data = Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default());
}
pfi.set_inline_data();
@@ -5526,10 +5523,8 @@ impl SetDisks {
);
let tier_free_version_source_lookup_started = rustfs_io_metrics::put_stage_timer();
let all_commit_disks_local = commit_disks.iter().flatten().all(|disk| disk.is_local());
let reuse_rename_data_source_for_tier_free_version = tier_free_version_source_reuse_rename_data_enabled()
&& (all_commit_disks_local
|| commit_set.tier_free_version_rename_data_source_capable()
&& (commit_set.tier_free_version_rename_data_source_capable()
|| tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled());
let mut put_tier_free_version_source = if commit_put_tier_free_version_id.is_some()
&& commit_tier_free_version_receipt_sink.is_some()
@@ -5627,6 +5622,7 @@ impl SetDisks {
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
}
let mut rename_guard_release = None;
let mut committed_tail_guard = None;
let mut needs_immediate_heal = false;
let mut tail_owns_tmp_cleanup = false;
rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -5635,11 +5631,9 @@ impl SetDisks {
);
let lock_held_tail_handoff_started = rustfs_io_metrics::put_stage_timer();
if let Ok(rename_commit) = rename_result.as_mut() {
if rename_commit.tail_drain.is_none() || !rename_commit.old_current_source_all_successful_checked {
commit_set.observe_tier_free_version_rename_data_source_capability(
rename_commit.tail_drain.is_none() && rename_commit.old_current_source_all_successful_checked,
);
}
commit_set.observe_tier_free_version_rename_data_source_capability(
rename_commit.tail_drain.is_none() && rename_commit.old_current_source_all_successful_checked,
);
commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks);
// Install the tail watcher before any post-commit await. The
// latch keeps namespace guards through their prior handoff point.
@@ -5658,7 +5652,10 @@ impl SetDisks {
.or_else(|| commit_version_suspended.then(Uuid::nil))
.map(|version_id| version_id.to_string());
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let object_lock_guard = _object_lock_guard.take();
let object_lock_guard = _object_lock_guard
.take()
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)));
committed_tail_guard = object_lock_guard.clone();
let publication_guard = _publication_guard.take();
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
@@ -5707,12 +5704,7 @@ impl SetDisks {
decommission_object_lock_guard,
decommission_capacity_guard,
),
targets,
old_current_source_all_successful_checked,
all_online_local_succeeded| {
cleanup_set.observe_tier_free_version_rename_data_source_capability(
old_current_source_all_successful_checked,
);
targets| {
drop(object_lock_guard);
drop(write_context);
drop(publication_guard);
@@ -5739,10 +5731,7 @@ impl SetDisks {
)
.await;
let tmp_delete_started = rustfs_io_metrics::put_stage_timer();
if zero_target_cleanup
&& (all_online_local_succeeded
|| rename_tail_cleanup_zero_target_tmp_delete_skip_enabled())
{
if zero_target_cleanup && rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() {
rustfs_io_metrics::record_put_object_stage_duration(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP,
1.0,
@@ -5972,6 +5961,13 @@ impl SetDisks {
lock_held_post_rename_started,
);
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
if let Some(guard) = committed_tail_guard.take() {
// The committed tail still owns every physical rename. Convert
// only after metadata invalidation and before acknowledging PUT.
// Unsupported peers keep their exclusive leases and the tail
// retains ownership until its physical work has drained.
let _ = guard.lock().await.downgrade_committed_tail().await;
}
if let Some(release) = rename_guard_release.take() {
let _ = release.send(true);
}
@@ -11065,9 +11061,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if let Some(err) = &gerr
&& goi.name.is_empty()
{
if (is_err_object_not_found(err) || is_err_version_not_found(err))
&& should_force_delete_marker_for_missing_version(&opts)
{
if should_force_delete_marker_for_missing_version(&opts) {
version_found = false;
} else {
if is_err_object_not_found(err) {
@@ -22177,6 +22171,52 @@ mod put_object_tmp_cleanup_tests {
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
temp_env::async_with_vars(
[
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
],
async {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "put-early-ack-tail-read";
let object = "early-ack-tail-read-object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let put_set = Arc::clone(&set_disks);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
put_set
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
.await
.expect("tail disk should pause during rename");
put.await
.expect("PUT task should join")
.expect("PUT should be acknowledged at quorum");
let result = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await;
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
let mut body = Vec::new();
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
.await
.expect("object body should not wait for the tail")
.expect("object body should drain");
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
rename_barrier.release();
drop(rename_barrier);
},
)
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
+12 -125
View File
@@ -599,33 +599,6 @@ impl ECStore {
Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta))
}
/// Snapshot only the object sets participating in this request. The sampler
/// resolves their physical leaves before Heal enters namespace/cache locks.
pub fn physical_heal_plan(&self, object: &str, opts: &HealOpts) -> Result<Option<crate::disk::PhysicalResourcePlan>> {
if let Some(index) = opts.pool
&& index >= self.pools.len()
{
return Err(invalid_heal_pool_index(index, self.pools.len()));
}
let object = rustfs_utils::path::encode_dir_object_ref(object);
let mut selected: Option<crate::disk::PhysicalResourcePlan> = None;
for (index, pool) in self.pools.iter().enumerate() {
if opts.pool.is_some_and(|requested| requested != index) {
continue;
}
let set = pool.get_disks_for_heal_object(&object, opts)?;
let Some(plan) = set.physical_heal_admission.plan() else { return Ok(None) };
if let Some(selected) = &mut selected {
if selected.union_with(&plan).is_err() {
return Ok(None);
}
} else {
selected = Some(plan);
}
}
Ok(selected)
}
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
match opts.pool {
Some(pool_idx) => Ok(vec![
@@ -1636,25 +1609,21 @@ mod tests {
}
}
async fn minimal_heal_pool(pool_idx: usize, set_count: usize) -> Arc<Sets> {
let format = FormatV3::new(set_count, 1);
let endpoints = (0..set_count)
.map(|set_idx| {
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx + set_idx * 2);
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
endpoint.set_pool_index(pool_idx);
endpoint.set_set_index(set_idx);
endpoint.set_disk_index(0);
endpoint
})
.collect::<Vec<_>>();
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
let format = FormatV3::new(1, 1);
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
endpoint.set_pool_index(pool_idx);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
Sets::new(
vec![None; set_count],
vec![None],
&PoolEndpoints {
legacy: false,
set_count,
set_count: 1,
drives_per_set: 1,
endpoints: Endpoints::from(endpoints),
endpoints: Endpoints::from(vec![endpoint]),
cmd_line: String::new(),
platform: String::new(),
},
@@ -2482,7 +2451,7 @@ mod tests {
ECStore {
id: Uuid::new_v4(),
disk_map: HashMap::new(),
pools: vec![minimal_heal_pool(0, 1).await, minimal_heal_pool(1, 1).await],
pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
peer_sys: S3PeerSys {
clients: Vec::new(),
pools_count: 2,
@@ -2497,88 +2466,6 @@ mod tests {
}
}
#[tokio::test]
async fn physical_heal_plan_uses_the_real_pool_set_and_directory_placement() {
use crate::disk::PhysicalDiskBinding;
use rustfs_concurrency::adaptive::{
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
};
use std::num::NonZeroU16;
let mut store = minimal_heal_store().await;
store.pools[0] = minimal_heal_pool(0, 2).await;
let pool = &store.pools[0];
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
for (set, gate) in pool.disk_set.iter().zip([&a, &b]) {
let binding = PhysicalDiskBinding::default();
binding.set_test_capacity(vec![gate.clone()]).expect("physical fixture");
set.physical_heal_admission
.set_test_plan(binding.plan().expect("live").resource_plan());
}
let node = NodeAdmission::new(NonZeroU16::MIN);
node.update(
&mut FeedbackController::new(),
Some(FeedbackObservation {
generation: 1,
max_concurrency: 1,
pressure: ResourcePressure::Unknown,
completed_units: None,
}),
);
let directory = (0..100)
.map(|n| format!("dir{n}/"))
.find(|name| {
store.pools[0].get_disks_by_key(name).set_index
!= store.pools[0].get_disks_by_key(&encode_dir_object(name)).set_index
})
.expect("different directory placement");
for object in ["plain-object", directory.as_str()] {
for set in [None, Some(0), Some(1)] {
let opts = HealOpts {
pool: Some(0),
set,
..Default::default()
};
let expected = store.pools[0]
.get_disks_for_heal_object(&encode_dir_object(object), &opts)
.expect("placement")
.set_index;
let permit = store
.physical_heal_plan(object, &opts)
.expect("valid selection")
.expect("known set")
.try_acquire(&node)
.expect("physical admission");
assert_eq!((a.snapshot().active, b.snapshot().active), if expected == 0 { (1, 0) } else { (0, 1) });
drop(permit);
}
}
assert!(
store
.physical_heal_plan("object", &HealOpts::default())
.expect("valid all pools")
.is_none(),
"an undiscovered participating pool cannot be skipped"
);
for opts in [
HealOpts {
pool: Some(2),
..Default::default()
},
HealOpts {
pool: Some(0),
set: Some(2),
..Default::default()
},
] {
assert!(matches!(
store.physical_heal_plan("object", &opts),
Err(StorageError::InvalidArgument(..))
));
}
}
async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
let missing_disk = target_set.disks.read().await[0]
-197
View File
@@ -5340,203 +5340,6 @@ mod tests {
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn conditional_multipart_must_not_commit_against_an_older_readable_pool() {
use crate::storage_api_contracts::multipart::CompletePart;
use crate::storage_api_contracts::object::HTTPPreconditions;
let temp_dir = tempfile::tempdir().expect("create conditional multipart test directory");
let (_ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "conditional-mpu-unreadable", &[4, 4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("conditional-mpu-{}", uuid::Uuid::new_v4().simple());
let object = "object";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create conditional multipart test bucket");
let mut old_body = PutObjReader::from_vec(b"older-pool".to_vec());
let old = store.pools[0]
.put_object(
&bucket,
object,
&mut old_body,
&ObjectOptions {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
..Default::default()
},
)
.await
.expect("write old object to staging pool");
let upload = store.pools[0]
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
.await
.expect("create multipart upload in pool 0");
let mut part_body = PutObjReader::from_vec(b"stale-writer".to_vec());
let part = store.pools[0]
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_body, &ObjectOptions::default())
.await
.expect("stage multipart part in pool 0");
let mut new_body = PutObjReader::from_vec(b"newer-pool".to_vec());
let new = store.pools[1]
.put_object(
&bucket,
object,
&mut new_body,
&ObjectOptions {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
..Default::default()
},
)
.await
.expect("write newer object to pool 1");
let old_etag = old.etag.clone().expect("old object must have an ETag");
let before = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
let mut metadata = Vec::new();
for disk_index in 0..4 {
let path = temp_dir
.path()
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/xl.meta"));
let original = tokio::fs::read(&path)
.await
.expect("save original object metadata before fault injection");
metadata.push((path, original));
}
let mut fault_error = None;
for (path, _) in &metadata {
if let Err(err) = tokio::fs::write(path, b"invalid object metadata").await {
fault_error = Some(err);
break;
}
}
let mut fault_contents = Vec::new();
if fault_error.is_none() {
for (path, _) in &metadata {
fault_contents.push(tokio::fs::read(path).await);
}
store.pools[1]
.get_disks_by_key(object)
.invalidate_get_object_metadata_cache(&bucket, object)
.await;
}
let unreadable = if fault_error.is_none() {
Some(
store.pools[1]
.get_object_info(
&bucket,
object,
&ObjectOptions {
no_lock: true,
metadata_cache_safe: false,
suppress_read_repair: true,
..Default::default()
},
)
.await,
)
} else {
None
};
let completed = if fault_error.is_none() {
let completion_store = Arc::clone(&store);
let completion_bucket = bucket.clone();
let upload_id = upload.upload_id.clone();
let part_etag = part.etag.clone();
Some(
tokio::spawn(async move {
completion_store
.complete_multipart_upload(
&completion_bucket,
object,
&upload_id,
vec![CompletePart {
part_num: 1,
etag: part_etag,
..Default::default()
}],
&ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_match: Some(old_etag),
..Default::default()
}),
..Default::default()
},
)
.await
})
.await,
)
} else {
None
};
let mut restore_error = None;
for (path, original) in &metadata {
if let Err(err) = tokio::fs::write(path, original).await {
restore_error = Some(err);
}
}
store.pools[1]
.get_disks_by_key(object)
.invalidate_get_object_metadata_cache(&bucket, object)
.await;
let after = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
let recovered_body = match store
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
{
Ok(mut reader) => {
let mut body = Vec::new();
let result = reader
.read_to_end(&mut body)
.await
.map(|_| body)
.map_err(|err| err.to_string());
drop(reader);
result
}
Err(err) => Err(err.to_string()),
};
let parts = store
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
.await;
shutdown.cancel();
assert!(fault_error.is_none(), "failed to inject metadata fault: {fault_error:?}");
assert!(restore_error.is_none(), "failed to restore metadata: {restore_error:?}");
assert_eq!(fault_contents.len(), 4, "all four pool 1 metadata files must be inspected");
for observed in fault_contents {
assert_eq!(
observed.expect("read injected metadata"),
b"invalid object metadata",
"each pool 1 metadata file must contain the injected corruption"
);
}
assert_eq!(before.expect("latest object before fault").etag, new.etag);
assert_ne!(old.etag, new.etag, "fixture bodies must produce different ETags");
let read_error = unreadable
.expect("metadata fault should have been injected")
.expect_err("the newer pool must fail its object metadata read during the fault");
assert!(
!crate::error::is_err_object_not_found(&read_error) && !crate::error::is_err_version_not_found(&read_error),
"corrupt metadata must produce a non-not-found read error: {read_error:?}"
);
let completed = completed
.expect("completion should have been attempted")
.expect("completion task should not panic");
let Err(completion_error) = completed else {
panic!("stale If-Match must not commit while the newer object's metadata is unreadable");
};
assert_eq!(completion_error, read_error, "completion must propagate the unreadable pool's error");
assert_eq!(after.expect("newer object after recovery").etag, new.etag);
assert_eq!(recovered_body.expect("newer body after recovery"), b"newer-pool");
let parts = parts.expect("failed completion must leave the upload retryable");
assert_eq!(parts.parts.len(), 1);
assert_eq!(parts.parts[0].etag, part.etag);
}
async fn migrate_versioned_decommission_test_object(
store: &Arc<crate::store::ECStore>,
bucket: &str,
+1 -32
View File
@@ -1292,37 +1292,6 @@ impl ECStore {
);
let rx = rx.child_token();
if self.pools.len() == 1 && self.pools[0].disk_set.len() == 1 {
rustfs_io_metrics::record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 1, 1);
let res = self.pools[0].disk_set[0].list_path(rx, opts, sender).await;
let mut outcome = ListPathRawOutcome::default();
let err = match res {
Ok(set_outcome) => {
outcome.limit_reached = set_outcome.limit_reached;
None
}
Err(err) => Some(err),
};
let errs = [err];
if is_all_not_found(&errs) {
return Ok(outcome);
}
if let Some(err) = errs.into_iter().flatten().next() {
return Err(err);
}
rustfs_io_metrics::record_stage_duration(
"store_list_objects_list_merged",
merge_started.elapsed().as_secs_f64() * 1000.0,
);
debug!(
set_pair_count = 1,
limit_reached = outcome.limit_reached,
error_count = 0,
"store list_merged finished"
);
return Ok(outcome);
}
let mut futures = Vec::new();
let mut inputs = Vec::new();
@@ -1842,7 +1811,7 @@ async fn gather_results(
if opts.versioned
&& is_object
&& entry
.xl_meta_ref()
.xl_meta()
.is_ok_and(|meta| !meta.versions.is_empty() && meta.versions.iter().all(|version| version.header.free_version()))
{
continue;
+33 -57
View File
@@ -458,8 +458,8 @@ pub(crate) mod utils;
use peer::init_local_peer;
pub use peer::{
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
@@ -498,17 +498,6 @@ pub struct ECStore {
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
}
impl Drop for ECStore {
/// Clear the instance context's `BucketMetadataSys` when its backing store
/// is dropped. Without this, the shared bootstrap context retains a
/// `BucketMetadataSys` whose `Weak<ECStore>` is dangling after the store
/// goes away (e.g. between sequential test fixtures that share the
/// bootstrap), and the next `object_store()` call panics.
fn drop(&mut self) {
self.ctx.clear_bucket_metadata_sys_if_stale();
}
}
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
@@ -1399,18 +1388,6 @@ impl crate::storage_api_contracts::object::ObjectIO for ECStore {
}
impl ECStore {
pub(crate) fn all_disks_local_and_online(&self) -> bool {
!self.disk_map.is_empty()
&& self.disk_map.values().all(|disks| {
!disks.is_empty()
&& disks.iter().all(|disk| {
disk.as_ref().is_some_and(|d| {
d.is_local() && d.runtime_state() == crate::disk::health_state::RuntimeDriveHealthState::Online
})
})
})
}
/// `put_object` plus the rename_data old-size backfill
/// (rustfs/backlog#1009); see `SetDisks::put_object_with_old_current_size`.
/// Post-write hooks (immediate ILM transition enqueue, list-cache
@@ -1423,36 +1400,19 @@ impl ECStore {
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
let result = self.handle_put_object(bucket, object, data, opts).await;
let result = match self.handle_put_object(bucket, object, data, opts).await {
Ok((object_info, old_current_size)) => {
enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
.await
.map(|object_info| (object_info, old_current_size))
}
Err(err) => Err(err),
};
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
result
}
#[allow(clippy::too_many_arguments)]
pub async fn copy_object_with_old_current_size(
&self,
src_bucket: &str,
src_object: &str,
dst_bucket: &str,
dst_object: &str,
src_info: &mut ObjectInfo,
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
let (oi, old_current_size) = box_copy_object_handler(move || {
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
})
.await?;
let result = enqueue_transition_after_write(self, Ok(oi), LcEventSrc::S3CopyObject, dst_opts)
.await
.map(|oi| (oi, old_current_size));
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(dst_bucket, 1);
}
result
}
}
lazy_static! {
@@ -1530,9 +1490,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<ObjectInfo> {
self.copy_object_with_old_current_size(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
.await
.map(|(oi, _)| oi)
let result = enqueue_transition_after_write(
self,
box_copy_object_handler(move || {
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
})
.await,
LcEventSrc::S3CopyObject,
dst_opts,
)
.await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(dst_bucket, 1);
}
result
}
#[instrument(skip(self))]
@@ -1792,10 +1763,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let result = self
.clone()
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
.await;
let result = enqueue_transition_after_write(
self.as_ref(),
self.clone()
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
.await,
LcEventSrc::S3CompleteMultipartUpload,
opts,
)
.await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
+15 -62
View File
@@ -612,7 +612,7 @@ impl ECStore {
}
if opts.http_preconditions.is_some()
&& !opts.data_movement
&& let Some(err) = self.check_write_precondition(bucket, object, &opts, false).await
&& let Some(err) = self.check_write_precondition(bucket, object, &opts).await
{
return Err(err);
}
@@ -640,19 +640,6 @@ impl ECStore {
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
if self.single_pool() && !self.is_suspended(0).await && !self.is_pool_rebalancing(0).await {
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
.await;
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
let lock_object = encode_dir_object(object);
let res = self
.run_external_decommission_capacity_object_mutation(0, bucket, &lock_object, object, opts, |opts| async move {
self.pools[0].new_multipart_upload(bucket, object, &opts).await
})
.await?;
return Ok((res, 0, expected_bucket_incarnation_id));
}
for (idx, pool) in self.pools.iter().enumerate() {
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
continue;
@@ -1154,33 +1141,12 @@ impl ECStore {
Ok(())
}
async fn check_write_precondition(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
require_all_pools: bool,
) -> Option<StorageError> {
async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option<StorageError> {
let mut lookup_opts = opts.clone();
let http_preconditions = lookup_opts.http_preconditions?;
lookup_opts.http_preconditions = None;
lookup_opts.no_lock = true;
let oi = if require_all_pools {
lookup_opts.metadata_cache_safe = false;
lookup_opts.suppress_read_repair = true;
if let Err(err) = check_object_args(bucket, object) {
return Some(err);
}
let encoded_object = encode_dir_object(object);
self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
.await
.and_then(|(info, _)| {
lookup_opts.precondition_check(&info)?;
Ok(info)
})
} else {
self.get_object_info_for_delete(bucket, object, &lookup_opts).await
};
let oi = self.get_object_info_for_delete(bucket, object, &lookup_opts).await;
crate::set_disk::SetDisks::evaluate_write_precondition(oi, &http_preconditions, bucket, object, opts)
}
@@ -1201,22 +1167,19 @@ impl ECStore {
let opts = &opts;
if self.single_pool() {
let result = self.pools[0]
return self.pools[0]
.clone()
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
.await;
return enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await;
}
if opts.http_preconditions.is_some() && !opts.data_movement {
// Preserve explicitly scoped internal reads and borrowed-lock callers.
let require_all_pools =
!opts.replication_request && !opts.no_lock && !opts.skip_decommissioned && !opts.skip_rebalancing;
if let Some(err) = self.check_write_precondition(bucket, object, opts, require_all_pools).await {
return Err(err);
}
if opts.http_preconditions.is_some()
&& !opts.data_movement
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
{
return Err(err);
}
let result = match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
Ok(pool_idx) => {
self.pools[pool_idx]
.clone()
@@ -1227,29 +1190,21 @@ impl ECStore {
// the pool that published it; other pools answer InvalidUploadID.
Err(err) if is_err_invalid_upload_id(&err) => {
let mut missing = err;
let mut found = None;
for pool_idx in self.existing_multipart_pool_order().await {
match self.pools[pool_idx]
.clone()
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts)
.await
{
Ok(info) => {
found = Some(Ok(info));
break;
}
Ok(info) => return Ok(info),
Err(err) if is_err_invalid_upload_id(&err) => missing = err,
Err(err) => {
found = Some(Err(err));
break;
}
Err(err) => return Err(err),
}
}
found.unwrap_or(Err(missing))
Err(missing)
}
Err(err) => Err(err),
};
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await
}
}
#[cfg(all(test, feature = "test-util"))]
@@ -1371,7 +1326,6 @@ impl ECStore {
// Data movement already owns the pool-meta write lease. Forward its
// loss signal into SetDisks so commit fencing observes the same lease
// without trying to reacquire the namespace.
let mut transition_opts = opts.clone();
let result = self
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
target_pool_idx,
@@ -1387,9 +1341,8 @@ impl ECStore {
)
.await;
drop(publication_guard);
transition_opts.no_lock = request_opts.no_lock;
let result =
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, &transition_opts).await;
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, request_opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
+22 -106
View File
@@ -31,7 +31,7 @@ use crate::bucket::lifecycle::{
};
use crate::bucket::metadata_sys::{
acquire_bucket_metadata_transaction_read_lock_in, get_cached_bucket_incarnation_id_in, get_delete_config_from_disk_in,
get_object_lock_and_quota_config_from_disk_with_guard_in, get_object_lock_config_and_incarnation_from_disk_with_guard_in,
get_object_lock_config_and_incarnation_from_disk_in,
};
use crate::bucket::object_lock::objectlock_sys::{
check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
@@ -3162,7 +3162,7 @@ impl ECStore {
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
let (state, bucket_incarnation_id, config_revision) =
get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
}
@@ -3177,30 +3177,6 @@ impl ECStore {
)))
}
async fn object_lock_config_snapshot_for_single_put(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
check_valid_bucket_name(bucket)?;
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
let (state, bucket_incarnation_id, config_revision, quota_res) =
get_object_lock_and_quota_config_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
}
let snapshot = ObjectLockConfigSnapshot::for_guarded_store_bucket(
self.id,
bucket,
bucket_incarnation_id,
config_revision,
state,
lifecycle_guard,
metadata_guard,
);
if !self.ctx.lock_manager().is_disabled() {
snapshot.set_fresh_put_quota(quota_res);
}
Ok(Arc::new(snapshot))
}
pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
&self,
bucket: &str,
@@ -3211,7 +3187,7 @@ impl ECStore {
}
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
let (state, bucket_incarnation_id, config_revision) =
get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&metadata_guard)).await?;
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
}
@@ -4676,7 +4652,7 @@ impl ECStore {
.expected_bucket_incarnation_id
.ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
if opts.object_lock_config_snapshot.is_none() {
opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot_for_single_put(bucket).await?);
opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
}
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
@@ -4765,7 +4741,6 @@ impl ECStore {
self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor))
.await;
let expected_data_bytes = usize::try_from(data.size()).ok();
let mut transition_opts = opts.clone();
let result = self
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
idx,
@@ -4797,8 +4772,7 @@ impl ECStore {
},
)
.await;
transition_opts.no_lock = request_opts.no_lock;
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, &transition_opts).await;
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
if result.is_ok() {
list_objects::observe_scanner_namespace_mutations(bucket, 1);
}
@@ -4814,7 +4788,7 @@ impl ECStore {
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
let idx = self
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
.await?;
@@ -4823,16 +4797,11 @@ impl ECStore {
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
}
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
let result = self.pools[idx]
.put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
.await;
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
match result {
Ok((object_info, old_current_size)) => {
Box::pin(enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, &opts))
.await
.map(|object_info| (object_info, old_current_size))
}
Err(err) => Err(err),
}
result
}
#[instrument(level = "trace", skip(self))]
@@ -4890,7 +4859,7 @@ impl ECStore {
src_info: &mut ObjectInfo,
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
) -> Result<ObjectInfo> {
check_copy_obj_args(src_bucket, src_object)?;
if src_bucket == dst_bucket && src_object == dst_object {
// In-place copies (metadata replace, restore) rewrite a key that
@@ -4969,17 +4938,9 @@ impl ECStore {
};
if cp_src_dst_same {
let pool_idx = if self.single_pool() {
0
} else {
self.get_latest_accessible_object_info_with_idx(
src_bucket,
&src_object,
&version_aware_lookup_opts(src_opts, true),
)
.await?
.1
};
let (_, pool_idx) = self
.get_latest_accessible_object_info_with_idx(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
.await?;
let source_pool_writable = !self.is_suspended(pool_idx).await && !self.is_pool_rebalancing(pool_idx).await;
if source_pool_writable
@@ -5000,8 +4961,7 @@ impl ECStore {
.await
},
)
.await
.map(|oi| (oi, None));
.await;
}
if source_pool_writable && !dst_opts.versioned && src_opts.version_id.is_none() {
@@ -5029,8 +4989,7 @@ impl ECStore {
.await
},
)
.await
.map(|oi| (oi, None));
.await;
}
// Transitioned object self-copy: restore from tier into the same pool.
let mut put_opts = ObjectOptions {
@@ -5058,7 +5017,7 @@ impl ECStore {
return if let Some(reader) = src_info.put_object_reader.as_mut() {
let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
let result = self.pools[pool_idx]
.put_object_with_old_current_size(dst_bucket, &dst_object, reader, &put_opts)
.put_object(dst_bucket, &dst_object, reader, &put_opts)
.await;
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
result
@@ -5102,7 +5061,7 @@ impl ECStore {
}
let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
let result = self.pools[pool_idx]
.put_object_with_old_current_size(dst_bucket, &dst_object, reader, &put_opts)
.put_object(dst_bucket, &dst_object, reader, &put_opts)
.await;
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
return result;
@@ -5122,14 +5081,11 @@ impl ECStore {
.await
},
)
.await
.map(|oi| (oi, None));
.await;
}
}
let pool_idx = if self.single_pool() && !self.is_suspended(0).await && !self.is_pool_rebalancing(0).await {
0
} else if dst_opts.no_lock {
let pool_idx = if dst_opts.no_lock {
self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
} else {
self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
@@ -5162,7 +5118,7 @@ impl ECStore {
if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
let result = self.pools[pool_idx]
.put_object_with_old_current_size(dst_bucket, dst_object_name, put_object_reader, &put_opts)
.put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
.await;
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
return result;
@@ -5394,9 +5350,8 @@ impl ECStore {
opts.add_namespace_lock_guard(guard);
}
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
let guard_ref = _bucket_lifecycle_guard.as_deref().unwrap_or_else(|| metadata_guard.as_ref());
let (state, incarnation_id, config_revision) =
get_delete_config_from_disk_in(&self.ctx, bucket, Some(guard_ref), opts.lifecycle_expiry_policy.as_ref()).await?;
get_delete_config_from_disk_in(&self.ctx, bucket, None, opts.lifecycle_expiry_policy.as_ref()).await?;
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
self.id,
bucket,
@@ -5636,44 +5591,6 @@ impl ECStore {
return Ok(obj);
}
if self.single_pool()
&& !opts.data_movement
&& opts.http_preconditions.is_none()
&& !opts.replication_request
&& !opts.version_suspended
&& !opts.delete_marker
&& !self.is_suspended(0).await
&& !self.is_pool_rebalancing(0).await
{
let pool = self.pools[0].clone();
match self
.run_external_decommission_capacity_object_delete(0, bucket, object, object, opts.clone(), |opts| async move {
pool.delete_object(bucket, object, opts).await
})
.await
{
Ok(mut res) => {
#[cfg(test)]
if creates_latest_marker {
pause_versioned_delete_marker_after_commit(bucket, object).await;
}
res.name = decode_dir_object(res.name.as_str());
return Ok(res);
}
Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
Err(err) => {
if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
return Ok(ObjectInfo {
bucket: bucket.to_owned(),
name: decode_dir_object(object),
..Default::default()
});
}
return Err(err);
}
}
}
// Determine which pool contains it
let existing_pool_info = if creates_latest_marker {
self.get_pool_info_for_delete_marker(bucket, object, &gopts).await
@@ -9986,7 +9903,6 @@ mod tests {
copy_store
.handle_copy_object(bucket, source, bucket, target, &mut source_info, &ObjectOptions::default(), &options)
.await
.map(|(oi, _)| oi)
});
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
tokio::time::timeout_at(deadline, barrier.wait_until_paused())
+3 -128
View File
@@ -99,9 +99,6 @@ impl ECStore {
fi: FileInfo,
expected: Uuid,
) -> DiskResult<()> {
if crate::disk::remote_io_admission::active() {
local_disk_candidate(&self.ctx, disk_ref).await?;
}
let disk_ref = disk_ref.to_owned();
let path = target.1.to_owned();
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
@@ -120,9 +117,6 @@ impl ECStore {
opts: DeleteOptions,
expected: Uuid,
) -> DiskResult<()> {
if crate::disk::remote_io_admission::active() {
local_disk_candidate(&self.ctx, disk_ref).await?;
}
let disk_ref = disk_ref.to_owned();
let path = target.1.to_owned();
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
@@ -150,9 +144,6 @@ impl ECStore {
if expected.is_nil() || opts.undo_write || self.ctx.lock_manager().is_disabled() {
return Err(DiskError::other("invalid incarnation-bound heal deletion"));
}
if crate::disk::remote_io_admission::active() {
local_disk_candidate(&self.ctx, disk_ref).await?;
}
let fence = Arc::new(
self.acquire_bucket_incarnation_fence(target.0, expected)
.await
@@ -309,9 +300,7 @@ impl ECStore {
// The optional ID is a cold lookup to cache only after final admission.
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
let map = ctx.local_disk_map();
let path_disk = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned();
if let Some(disk) = path_disk {
admit_rpc_disk(&disk)?;
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
return Ok((disk, None));
}
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
@@ -320,16 +309,11 @@ async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> Dis
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
if let Some(disk) = cached_disk
&& matches!(disk.as_ref(), Disk::Local(_))
&& disk.get_disk_id().await? == Some(disk_id)
{
admit_rpc_disk(&disk)?;
if disk.get_disk_id().await? == Some(disk_id) {
return Ok((disk, None));
}
return Ok((disk, None));
}
}
if crate::disk::remote_io_admission::active() {
return Err(DiskError::Io(std::io::Error::from(std::io::ErrorKind::WouldBlock)));
}
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
// Disk identity may perform format I/O. No registry guard spans this await.
for disk in disks {
@@ -340,17 +324,6 @@ async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> Dis
Err(DiskError::DiskNotFound)
}
fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
use crate::disk::remote_io_admission;
if remote_io_admission::active() {
let binding = disk
.physical_admission()
.ok_or_else(|| DiskError::Io(std::io::Error::from(std::io::ErrorKind::WouldBlock)))?;
remote_io_admission::admit(&binding).map_err(DiskError::Io)?;
}
Ok(())
}
async fn admit_local_disk(
ctx: &Arc<InstanceContext>,
disk: &DiskStore,
@@ -510,18 +483,6 @@ pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
None
}
/// Resolve and admit an authenticated RPC's target before its storage locks.
/// Unknown UUID mappings defer background work instead of probing unrelated disks.
pub async fn find_local_disk_by_ref_for_rpc(disk_ref: &str) -> DiskResult<Option<DiskStore>> {
use crate::disk::remote_io_admission;
if !remote_io_admission::active() {
return Ok(find_local_disk_by_ref(disk_ref).await);
}
let (disk, _) = local_disk_candidate(&crate::runtime::global::current_ctx(), disk_ref).await?;
Ok(Some(disk))
}
pub async fn all_local_disk_path() -> Vec<String> {
runtime_sources::local_disk_paths().await
}
@@ -882,92 +843,6 @@ mod tests {
}
}
#[tokio::test]
async fn remote_background_target_lookup_defers_before_io_and_reuses_multi_phase_admission() {
use crate::disk::{RemoteBackgroundIo, with_remote_background_io};
use rustfs_concurrency::adaptive::{
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
};
use std::num::NonZeroU16;
let ctx = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("target directory");
let id = Uuid::new_v4();
let disk = target_disk(&ctx, root.path(), id).await;
disk.make_volume("bucket").await.expect("fixture volume");
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
disk.physical_admission()
.expect("local binding")
.set_test_capacity(vec![gate.clone()])
.expect("physical device");
let node = NodeAdmission::new(NonZeroU16::MIN);
node.update(
&mut FeedbackController::new(),
Some(FeedbackObservation {
generation: 1,
max_concurrency: 1,
pressure: ResourcePressure::Unknown,
completed_units: None,
}),
);
let path = disk.endpoint().to_string();
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
assert!(
matches!(disk.read_all("bucket", "object").await,
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
"no implicit target admission"
);
assert!(
matches!(local_disk_candidate(&ctx, &path).await,
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
"closed receiver capacity"
);
gate.set_limit(1);
assert!(
matches!(local_disk_candidate(&ctx, &id.to_string()).await,
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
"cold UUID cannot probe unrelated disks"
);
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
let (selected, _) = local_disk_candidate(&ctx, &path).await.expect("select known local path");
assert!(Arc::ptr_eq(&selected, &disk));
for body in [b"first phase".as_slice(), b"second phase".as_slice()] {
disk.write_all("bucket", "object", bytes::Bytes::copy_from_slice(body))
.await
.expect("write phase");
let (again, _) = local_disk_candidate(&ctx, &path)
.await
.expect("repeat target resolution at capacity one");
assert_eq!(again.read_all("bucket", "object").await.expect("read phase").as_ref(), body);
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
}
})
.await;
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
assert_eq!([node.completed(), gate.completed()], [Some(1), Some(1)]);
assert!(record_local_disk_id_if_active(&ctx, &disk, id).await);
let replacement = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
replacement
.physical_admission()
.expect("replacement binding")
.set_test_capacity(vec![gate.clone()])
.expect("replacement device");
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
assert!(
matches!(local_disk_candidate(&ctx, &id.to_string()).await,
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
"stale UUID must not resolve replacement data"
);
})
.await;
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
assert_eq!(
[node.completed(), gate.completed()],
[Some(1), Some(1)],
"identity checks alone are not work completion"
);
}
#[tokio::test]
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
let ctx = Arc::new(InstanceContext::new());
+2 -27
View File
@@ -837,29 +837,7 @@ impl ECStore {
object: &str,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
self.get_latest_object_info_with_pool_policy(bucket, object, opts, false)
.await
}
/// Observe every pool for a conditional-write preflight. This does not
/// hold the namespace locks needed to prove a condition at publication.
pub(super) async fn get_latest_object_info_for_conditional_write(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
self.get_latest_object_info_with_pool_policy(bucket, object, opts, true).await
}
async fn get_latest_object_info_with_pool_policy(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
require_all_pools: bool,
) -> Result<(ObjectInfo, usize)> {
let suspended_pools = if opts.skip_decommissioned && !require_all_pools {
let suspended_pools = if opts.skip_decommissioned {
let pool_meta = self.pool_meta.read().await;
Some(
(0..self.pools.len())
@@ -875,7 +853,7 @@ impl ECStore {
continue;
}
if opts.skip_rebalancing && !require_all_pools && self.is_pool_rebalancing(idx).await {
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
continue;
}
@@ -895,9 +873,6 @@ impl ECStore {
});
}
Err(e) => {
if require_all_pools && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
return Err(e);
}
candidates.push(LatestObjectInfoCandidate {
info: None,
idx,
-47
View File
@@ -2070,53 +2070,6 @@ mod test {
assert!(meta.versions.iter().all(|version| !version.header.free_version()));
}
#[test]
fn bytes_count_toward_listing_limit_matches_fileinfo_visibility() {
let mut meta = FileMeta::new();
let mut obj = FileInfo::new("object", 2, 2);
obj.version_id = Some(Uuid::new_v4());
obj.mod_time = Some(OffsetDateTime::now_utc());
meta.add_version(obj.clone()).expect("add object version");
let obj_bytes = meta.marshal_msg().expect("marshal object");
assert!(FileMeta::bytes_count_toward_listing_limit(&obj_bytes));
let dm = FileInfo {
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
deleted: true,
mark_deleted: true,
mod_time: obj.mod_time.map(|t| t + time::Duration::seconds(1)),
..Default::default()
};
meta.delete_version(&dm).expect("add delete marker");
let dm_bytes = meta.marshal_msg().expect("marshal delete marker");
assert!(!FileMeta::bytes_count_toward_listing_limit(&dm_bytes));
let (_, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
source.version_id = Some(Uuid::new_v4());
let mut transitioned = FileMeta::new();
transitioned.add_version(source.clone()).expect("seed transitioned version");
let mut del_req = FileInfo {
name: "object".to_string(),
version_id: source.version_id,
mod_time: source.mod_time,
..Default::default()
};
del_req.set_tier_free_version_id(&Uuid::new_v4().to_string());
transitioned
.delete_version(&del_req)
.expect("delete transitioned version leaving free version");
assert!(!FileMeta::bytes_count_toward_listing_limit(
&transitioned.marshal_msg().expect("marshal free-only")
));
transitioned.add_version(obj).expect("add live object version");
assert!(FileMeta::bytes_count_toward_listing_limit(
&transitioned.marshal_msg().expect("marshal free + live object")
));
assert!(FileMeta::bytes_count_toward_listing_limit(b"corrupt-xl-meta"));
}
#[test]
fn add_version_filemata_uses_canonical_equal_time_order() {
let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");
-100
View File
@@ -324,63 +324,6 @@ impl FileMeta {
true
}
pub(crate) fn buffers_match_strict(self_buf: &[u8], other_buf: &[u8]) -> bool {
let Ok((s_meta, _)) = Self::parse_validated_xl_envelope(self_buf) else {
return false;
};
let Ok((o_meta, _)) = Self::parse_validated_xl_envelope(other_buf) else {
return false;
};
if s_meta.is_empty() || o_meta.is_empty() {
return s_meta.is_empty() && o_meta.is_empty();
}
let Ok((s_len, s_header_ver, _s_meta_ver, s_meta)) = Self::decode_validated_xl_headers(s_meta) else {
return false;
};
let Ok((o_len, o_header_ver, _o_meta_ver, o_meta)) = Self::decode_validated_xl_headers(o_meta) else {
return false;
};
if s_len != o_len {
return false;
}
let mut s_cur: Cursor<&[u8]> = Cursor::new(s_meta);
let mut o_cur: Cursor<&[u8]> = Cursor::new(o_meta);
for _ in 0..s_len {
let Ok((s_header_raw, _)) = Self::read_next_raw_slices(s_meta, &mut s_cur) else {
return false;
};
let Ok((o_header_raw, _)) = Self::read_next_raw_slices(o_meta, &mut o_cur) else {
return false;
};
if s_header_ver == XL_HEADER_VERSION
&& o_header_ver == XL_HEADER_VERSION
&& s_header_raw.len() == 38
&& o_header_raw.len() == 38
&& s_header_raw == o_header_raw
&& parse_canonical_v3_header_fields(s_header_raw).is_some()
{
continue;
}
let mut s_header = FileMetaVersionHeader::default();
if s_header.unmarshal_v(s_header_ver, s_header_raw).is_err() {
return false;
}
let mut o_header = FileMetaVersionHeader::default();
if o_header.unmarshal_v(o_header_ver, o_header_raw).is_err() {
return false;
}
if !Self::headers_match_strict(&s_header, &o_header) {
return false;
}
}
true
}
#[inline]
fn header_matches_canonical_v3_raw_strict(s_header: &FileMetaVersionHeader, buf: &[u8]) -> Option<bool> {
if buf.len() != 38 {
@@ -682,49 +625,6 @@ impl FileMeta {
}
}
pub fn bytes_count_toward_listing_limit(buf: &[u8]) -> bool {
let Ok((meta, _inline_buf)) = Self::parse_validated_xl_envelope(buf) else {
return true;
};
if meta.is_empty() {
return false;
}
let Ok((versions_len, header_ver, _meta_ver, meta)) = Self::decode_validated_xl_headers(meta) else {
return true;
};
if versions_len == 0 {
return false;
}
let mut cur = Cursor::new(meta);
let mut selected: Option<(FileMetaVersionHeader, &[u8])> = None;
let mut any_non_hidden = false;
for _ in 0..versions_len {
let Ok((header, ver_meta)) = Self::read_next_shallow_slice(meta, &mut cur, header_ver) else {
return true;
};
if header.version_type != VersionType::Delete && !header.free_version() {
any_non_hidden = true;
}
if selected.is_some() {
continue;
}
if header.free_version() && header.version_id.is_some_and(|id| !id.is_nil()) {
continue;
}
selected = Some((header, ver_meta));
}
let Some((header, ver_meta)) = selected else {
return false;
};
if !header.free_version() && header.version_type != VersionType::Delete {
return true;
}
if !any_non_hidden {
return false;
}
FileMetaVersion::decode_fileinfo_from_slice(ver_meta, "", "", true, false).is_err()
}
#[hotpath::measure(impl_type = "FileMeta")]
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut wr =
-197
View File
@@ -352,22 +352,6 @@ impl MetaCacheEntry {
(prefer, true)
}
pub fn matches_strict_borrowed(&mut self, other: &MetaCacheEntry) -> bool {
if self.name != other.name {
return false;
}
if other.is_dir() || self.is_dir() {
return other.is_dir() == self.is_dir();
}
if self.cached.is_none() && other.cached.is_none() {
return FileMeta::buffers_match_strict(&self.metadata, &other.metadata);
}
self.matches_strict(other)
}
pub fn matches_strict(&mut self, other: &MetaCacheEntry) -> bool {
if self.name != other.name {
return false;
@@ -1005,17 +989,6 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
Ok(())
}
pub async fn write_obj_buffered(&mut self, obj: &MetaCacheEntry) -> Result<()> {
self.init().await?;
Self::encode_obj(&mut self.buf, obj)?;
if self.buf.len() >= METACACHE_READ_CHUNK_SIZE {
self.flush().await?;
}
Ok(())
}
pub async fn close(&mut self) -> Result<()> {
self.close_with_limit_reached(false).await
}
@@ -1309,85 +1282,6 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
}
pub fn skip_current(&mut self) -> bool {
if self.has_current && self.init && self.err.is_none() {
self.has_current = false;
true
} else {
false
}
}
pub fn try_peek_buffered(&mut self) -> Option<Result<Option<MetaCacheEntry>>> {
if !self.init || self.err.is_some() || self.terminated {
return None;
}
let avail = &self.buf[self.pos..];
let &first = avail.first()?;
if first != 0xc3 {
return None;
}
let str_marker = *avail.get(1)?;
let (str_hdr, str_len) = match str_marker {
0xa0..=0xbf => (1usize, usize::from(str_marker & 0x1f)),
0xd9 => (2usize, usize::from(*avail.get(2)?)),
0xda => (3usize, usize::from(u16::from_be_bytes([*avail.get(2)?, *avail.get(3)?]))),
0xdb => (
5usize,
usize::try_from(u32::from_be_bytes([*avail.get(2)?, *avail.get(3)?, *avail.get(4)?, *avail.get(5)?])).ok()?,
),
_ => return None,
};
let bin_offset = 1usize.checked_add(str_hdr)?.checked_add(str_len)?;
let bin_marker = *avail.get(bin_offset)?;
let (bin_hdr, bin_len) = match bin_marker {
0xc4 => (2usize, usize::from(*avail.get(bin_offset + 1)?)),
0xc5 => (
3usize,
usize::from(u16::from_be_bytes([*avail.get(bin_offset + 1)?, *avail.get(bin_offset + 2)?])),
),
0xc6 => (
5usize,
usize::try_from(u32::from_be_bytes([
*avail.get(bin_offset + 1)?,
*avail.get(bin_offset + 2)?,
*avail.get(bin_offset + 3)?,
*avail.get(bin_offset + 4)?,
]))
.ok()?,
),
_ => return None,
};
let total_len = bin_offset.checked_add(bin_hdr)?.checked_add(bin_len)?;
if avail.len() < total_len {
return None;
}
let Ok(decoded) = from_utf8(&avail[1 + str_hdr..bin_offset]) else {
return None;
};
let name = decoded.to_owned();
let metadata = if bin_len == 0 {
Vec::new()
} else {
avail[bin_offset + bin_hdr..total_len].to_vec()
};
self.pos += total_len;
self.reset();
self.has_current = true;
Some(Ok(Some(MetaCacheEntry {
name,
metadata,
cached: None,
reusable: false,
})))
}
pub async fn skip(&mut self, size: usize) -> Result<()> {
self.check_init().await?;
@@ -4363,95 +4257,4 @@ mod tests {
.expect("resolve mixed empty");
assert_eq!(res_mixed_empty.cached.expect("cached").versions.len(), 1);
}
#[test]
fn matches_strict_borrowed_matches_shallow_headers_and_rejects_corrupt_or_divergent_buffers() {
let base_bytes = create_real_xlmeta().expect("create real xlmeta");
let mut peer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
// Mutate inner version payload (ver.meta) without changing shallow header.
let mut ver = FileMetaVersion::try_from(peer_fm.versions[1].meta.as_slice()).expect("decode version");
ver.object.as_mut().expect("object version").erasure_index += 1;
peer_fm.versions[1].meta = ver.marshal_msg().expect("marshal peer version");
let peer_bytes = peer_fm.marshal_msg().expect("marshal peer xlmeta");
assert_ne!(base_bytes, peer_bytes, "peer shard has distinct erasure index payload");
let mut e1 = MetaCacheEntry {
name: "obj-a".to_owned(),
metadata: base_bytes.clone(),
cached: None,
reusable: false,
};
let e1_clone = e1.clone();
let e2 = MetaCacheEntry {
name: "obj-a".to_owned(),
metadata: peer_bytes,
cached: None,
reusable: false,
};
assert!(e1.matches_strict_borrowed(&e2));
assert!(e1.matches_strict_borrowed(&e1_clone));
// Different shallow header must not match.
let mut newer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
newer_fm.versions[0].header.version_id = Some(Uuid::new_v4());
let newer_entry = MetaCacheEntry {
name: "obj-a".to_owned(),
metadata: newer_fm.marshal_msg().expect("marshal newer xlmeta"),
cached: None,
reusable: false,
};
assert!(!e1.matches_strict_borrowed(&newer_entry));
// Identical corrupt buffers must return false, never true via byte-equality shortcut.
let mut corrupt = MetaCacheEntry {
name: "obj-a".to_owned(),
metadata: b"XL2 corrupt-payload".to_vec(),
cached: None,
reusable: false,
};
let corrupt_peer = corrupt.clone();
assert!(!corrupt.matches_strict_borrowed(&corrupt_peer));
}
#[tokio::test]
async fn try_peek_buffered_and_skip_current_match_async_peek_and_skip() {
let entries: Vec<_> = (0..5)
.map(|i| MetaCacheEntry {
name: format!("buffered-key-{i}"),
metadata: vec![i as u8; 16],
cached: None,
reusable: false,
})
.collect();
let mut buf = Cursor::new(Vec::new());
let mut writer = MetacacheWriter::new(&mut buf);
writer.write(&entries).await.expect("write entries");
writer.close_with_limit_reached(false).await.expect("close stream");
let mut reader = MetacacheReader::new(Cursor::new(buf.into_inner()));
assert!(reader.try_peek_buffered().is_none(), "uninitialized reader must defer to async peek");
assert!(!reader.skip_current(), "no current entry before first peek");
let first = reader.peek().await.expect("first peek").expect("entry 0");
assert_eq!(first, entries[0]);
assert!(reader.skip_current(), "skip_current must consume the peeked entry");
assert!(!reader.skip_current(), "second skip_current without peek must return false");
for expected in &entries[1..] {
let next = reader
.try_peek_buffered()
.expect("remaining entries are already buffered")
.expect("valid entry")
.expect("present entry");
assert_eq!(&next, expected);
assert!(reader.skip_current());
}
assert!(
reader.try_peek_buffered().is_none(),
"stream terminator must defer to async peek for trailer handling"
);
assert!(reader.peek().await.expect("eof peek").is_none());
assert_eq!(reader.limit_reached(), Some(false));
}
}
-6
View File
@@ -88,12 +88,6 @@ pub enum HealObjectDisposition {
pub struct HealObjectReceipt {
pub identity: HealObjectIdentity,
pub disposition: HealObjectDisposition,
/// Data directory selected from quorum metadata during storage execution.
/// This does not establish lock ownership or identity at completion.
pub quorum_selected_data_dir: Option<Uuid>,
/// Storage completed its Deep check path for the selected metadata.
/// This does not certify all bytes, final identity, or scanner coverage.
pub deep_check_observed: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+1 -2
View File
@@ -84,7 +84,7 @@ rustfs-heal-contracts = { workspace = true }
rustfs-madmin = { workspace = true }
rustfs-utils = { workspace = true }
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
tokio-util = { workspace = true, features = ["io", "compat"] }
tonic = { workspace = true }
tracing = { workspace = true }
serde = { workspace = true, features = ["derive"] }
@@ -100,7 +100,6 @@ crc-fast = { workspace = true }
sha2 = { workspace = true }
[dev-dependencies]
rustfs-ecstore = { workspace = true, features = ["test-util"] }
rustfs-filemeta = { workspace = true }
serde_json = { workspace = true, features = ["raw_value"] }
rustfs-test-utils = { workspace = true }
+4 -33
View File
@@ -186,7 +186,7 @@ pub(crate) enum HealErrorClass {
/// Offline disk, missing volume, unmet quorum, transport failure,
/// slow-down or cancelled RPC: the same request can succeed later.
Transient,
/// Nonblocking admission/lock contention or a lock/RPC timeout.
/// Lock contention or a lock/RPC timeout (non-fatal `LockError`).
LockContention,
/// A storage grace window; `retry_not_before` is the absolute UNIX
/// deadline when storage reported one.
@@ -279,7 +279,8 @@ pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
}
}
/// Classify a disk error without losing the underlying I/O deferral kind.
/// Classify a disk error. Nonblocking local CAS and replacement leases report
/// lock contention as `WouldBlock`.
pub(crate) fn classify_disk_error(error: &DiskError) -> HealErrorClass {
if error.is_dangling_delete_grace() {
return HealErrorClass::DanglingGrace {
@@ -290,6 +291,7 @@ pub(crate) fn classify_disk_error(error: &DiskError) -> HealErrorClass {
DiskError::FileNotFound | DiskError::FileVersionNotFound => HealErrorClass::Absent,
DiskError::NoHealRequired => HealErrorClass::NoHealRequired,
DiskError::Timeout => HealErrorClass::LockContention,
DiskError::Io(error) if error.kind() == std::io::ErrorKind::WouldBlock => HealErrorClass::LockContention,
DiskError::DiskNotFound
| DiskError::VolumeNotFound
| DiskError::ErasureReadQuorum
@@ -304,14 +306,6 @@ pub(crate) fn classify_disk_error(error: &DiskError) -> HealErrorClass {
}
fn classify_io_error(error: &std::io::Error) -> HealErrorClass {
// Native memory admission and nonblocking CAS keep this kind when their
// errors cross disk, storage and direct I/O wrappers.
if error.kind() == std::io::ErrorKind::WouldBlock {
return HealErrorClass::LockContention;
}
if error.kind() == std::io::ErrorKind::OutOfMemory {
return HealErrorClass::Transient;
}
if DiskError::io_error_is_dangling_delete_grace(error) {
return HealErrorClass::DanglingGrace {
retry_not_before: dangling_deadline(DiskError::io_error_dangling_delete_retry_after(error)),
@@ -870,29 +864,6 @@ mod tests {
assert!(Error::Disk(DiskError::VolumeNotFound).is_recoverable_object_heal());
}
#[test]
fn resource_pressure_stays_retryable_across_error_wrappers() {
for (kind, expected) in [
(std::io::ErrorKind::WouldBlock, super::HealErrorClass::LockContention),
(std::io::ErrorKind::OutOfMemory, super::HealErrorClass::Transient),
] {
let pressure = || DiskError::Io(std::io::Error::new(kind, "background buffer memory budget exhausted"));
for error in [
Error::Disk(pressure()),
Error::Storage(EcstoreError::from(pressure())),
Error::Io(EcstoreError::from(pressure()).into()),
] {
assert_eq!(super::classify_heal_error(&error), expected);
assert!(error.is_recoverable_heal(), "resource pressure must preserve retry responsibility");
assert!(!error.is_object_absent());
}
}
for kind in [std::io::ErrorKind::Other, std::io::ErrorKind::PermissionDenied] {
let error = Error::Io(std::io::Error::new(kind, "background buffer memory budget exhausted"));
assert!(!error.is_recoverable_heal(), "display text must not override the typed error kind");
}
}
#[test]
fn classify_heal_error_table() {
use super::{HealErrorClass as C, classify_heal_error};
-187
View File
@@ -28,7 +28,6 @@ use rustfs_madmin::heal_commands::HealResultItem;
use serde::Serialize;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info};
const LOG_COMPONENT_HEAL: &str = "heal";
@@ -61,8 +60,6 @@ pub struct HealChannelProcessor {
response_sender: mpsc::UnboundedSender<HealChannelResponse>,
/// Response receiver
response_receiver: mpsc::UnboundedReceiver<HealChannelResponse>,
#[cfg(test)]
command_pause: Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>,
}
#[derive(Serialize)]
@@ -164,8 +161,6 @@ impl HealChannelProcessor {
heal_manager,
response_sender: response_tx,
response_receiver: response_rx,
#[cfg(test)]
command_pause: None,
}
}
@@ -234,19 +229,9 @@ impl HealChannelProcessor {
/// Start processing legacy and canonical-receipt heal channel requests.
pub async fn start_with_receipts(
&mut self,
receiver: HealChannelReceiver,
receipt_receiver: HealReceiptReceiver,
) -> Result<()> {
self.start_with_shutdown(receiver, receipt_receiver, CancellationToken::new())
.await
}
pub(crate) async fn start_with_shutdown(
&mut self,
mut receiver: HealChannelReceiver,
mut receipt_receiver: HealReceiptReceiver,
stop: CancellationToken,
) -> Result<()> {
info!(
target: "rustfs::heal::channel",
@@ -259,30 +244,10 @@ impl HealChannelProcessor {
let mut receipt_channel_open = true;
loop {
if stop.is_cancelled() {
receiver.close();
receipt_receiver.close();
while let Some(command) = receiver.recv().await {
Self::reject_stopped_command(command);
}
while let Some(HealReceiptCommand { response_tx, .. }) = receipt_receiver.recv().await {
let _ = response_tx.send(Err(Error::other("Heal channel is stopping").to_string()));
}
self.response_receiver.close();
while self.response_receiver.try_recv().is_ok() {}
break;
}
tokio::select! {
_ = stop.cancelled() => {}
command = receiver.recv() => {
match command {
Some(command) => {
if stop.is_cancelled() {
Self::reject_stopped_command(command);
continue;
}
#[cfg(test)]
self.pause_selected_command().await;
if let Err(e) = self.process_command(command).await {
error!(
target: "rustfs::heal::channel",
@@ -313,12 +278,6 @@ impl HealChannelProcessor {
receipt_channel_open = false;
continue;
};
if stop.is_cancelled() {
let _ = response_tx.send(Err(Error::other("Heal channel is stopping").to_string()));
continue;
}
#[cfg(test)]
self.pause_selected_command().await;
if let Err(e) = self.process_start_request(request, false, true, response_tx).await {
error!(
target: "rustfs::heal::channel",
@@ -360,26 +319,6 @@ impl HealChannelProcessor {
Ok(())
}
fn reject_stopped_command(command: HealChannelCommand) {
let error = Error::other("Heal channel is stopping").to_string();
match command {
HealChannelCommand::Start { response_tx, .. } => {
let _ = response_tx.send(Err(error));
}
HealChannelCommand::Query { response_tx, .. } | HealChannelCommand::Cancel { response_tx, .. } => {
let _ = response_tx.send(Err(error));
}
}
}
#[cfg(test)]
async fn pause_selected_command(&mut self) {
if let Some((started, release)) = self.command_pause.take() {
let _ = started.send(());
let _ = release.await;
}
}
/// Process heal command
async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
match command {
@@ -1061,8 +1000,6 @@ mod tests {
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
};
outcome.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".into(),
@@ -1147,8 +1084,6 @@ mod tests {
outcome.start();
for index in 0..256 {
outcome.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".into(),
@@ -2534,126 +2469,4 @@ mod tests {
assert_eq!(response.request_id, "missing-token");
assert!(response.error.unwrap_or_default().contains("Heal task not found"));
}
async fn stop_after_selected_start(canonical: bool) {
let manager = create_test_heal_manager();
let mut processor = HealChannelProcessor::new(manager.clone());
let (selected_tx, selected_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
processor.command_pause = Some((selected_tx, release_rx));
let (command_tx, command_rx) = mpsc::unbounded_channel();
let (receipt_tx, receipt_rx) = mpsc::unbounded_channel();
let stop = CancellationToken::new();
let worker_stop = stop.clone();
let worker = tokio::spawn(async move { processor.start_with_shutdown(command_rx, receipt_rx, worker_stop).await });
let request = HealChannelRequest {
id: "active-shutdown-command".to_string(),
bucket: "bucket".to_string(),
object_prefix: Some("active".to_string()),
source: HealRequestSource::Admin,
..Default::default()
};
let mut legacy_response = None;
let mut canonical_response = None;
if canonical {
let (response_tx, response_rx) = oneshot::channel();
receipt_tx.send(HealReceiptCommand { request, response_tx }).unwrap();
canonical_response = Some(response_rx);
} else {
let (response_tx, response_rx) = oneshot::channel();
command_tx.send(HealChannelCommand::Start { request, response_tx }).unwrap();
legacy_response = Some(response_rx);
}
tokio::time::timeout(Duration::from_secs(1), selected_rx)
.await
.unwrap()
.unwrap();
let pending = HealChannelRequest {
id: "queued-shutdown-command".to_string(),
bucket: "bucket".to_string(),
object_prefix: Some("pending".to_string()),
source: HealRequestSource::Admin,
..Default::default()
};
let (start_tx, start_rx) = oneshot::channel();
command_tx
.send(HealChannelCommand::Start {
request: pending.clone(),
response_tx: start_tx,
})
.unwrap();
let (query_tx, query_rx) = oneshot::channel();
command_tx
.send(HealChannelCommand::Query {
heal_path: "bucket/active".to_string(),
client_token: "active-shutdown-command".to_string(),
since_seq: None,
response_tx: query_tx,
})
.unwrap();
let (cancel_tx, cancel_rx) = oneshot::channel();
command_tx
.send(HealChannelCommand::Cancel {
heal_path: "bucket/active".to_string(),
client_token: "active-shutdown-command".to_string(),
response_tx: cancel_tx,
})
.unwrap();
let (receipt_response_tx, receipt_response_rx) = oneshot::channel();
receipt_tx
.send(HealReceiptCommand {
request: pending,
response_tx: receipt_response_tx,
})
.unwrap();
stop.cancel();
tokio::task::yield_now().await;
assert!(!worker.is_finished(), "stop cannot cancel the selected command");
release_tx.send(()).unwrap();
tokio::time::timeout(Duration::from_secs(1), worker)
.await
.unwrap()
.unwrap()
.unwrap();
if let Some(response) = legacy_response {
assert_eq!(response.await.unwrap().unwrap(), HealAdmissionResult::Accepted);
}
if let Some(response) = canonical_response {
let receipt = response.await.unwrap().unwrap();
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
assert_eq!(receipt.task_id, "active-shutdown-command");
}
let expected = Error::other("Heal channel is stopping").to_string();
assert_eq!(start_rx.await.unwrap().unwrap_err(), expected);
assert_eq!(query_rx.await.unwrap().unwrap_err(), expected);
assert_eq!(cancel_rx.await.unwrap().unwrap_err(), expected);
assert_eq!(receipt_response_rx.await.unwrap().unwrap_err(), expected);
assert!(command_tx.is_closed());
assert!(receipt_tx.is_closed());
assert_eq!(
manager
.get_task_status_for_path("bucket/active", "active-shutdown-command")
.await
.unwrap(),
HealTaskStatus::Pending,
"queued Cancel must not execute"
);
assert!(
manager
.get_task_status_for_path("bucket/pending", "queued-shutdown-command")
.await
.is_err(),
"queued starts must not be admitted"
);
manager.stop().await.unwrap();
}
#[tokio::test]
async fn owned_channel_stop_finishes_legacy_command_and_rejects_all_queued_variants() {
stop_after_selected_start(false).await;
}
#[tokio::test]
async fn owned_channel_stop_finishes_receipt_command_and_rejects_all_queued_variants() {
stop_after_selected_start(true).await;
}
}
+5 -34
View File
@@ -127,7 +127,6 @@ pub struct ErasureSetHealer {
replacement_task_id: Option<String>,
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
work_parent: Option<super::manager::HealWork>,
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
}
@@ -368,7 +367,6 @@ impl ErasureSetHealer {
replacement_task_id: None,
replacement_target_identities: None,
replacement_execution: None,
work_parent: None,
mainline_pacer: None,
}
}
@@ -383,11 +381,6 @@ impl ErasureSetHealer {
self
}
pub(super) fn with_work_parent(mut self, parent: Option<super::manager::HealWork>) -> Self {
self.work_parent = parent;
self
}
pub(crate) fn with_replacement_execution(mut self, execution: Option<Arc<super::storage::ReplacementExecution>>) -> Self {
self.replacement_execution = execution;
self
@@ -1356,8 +1349,6 @@ impl ErasureSetHealer {
};
(
HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Metadata,
bucket: RUSTFS_META_BUCKET.to_string(),
@@ -1729,8 +1720,6 @@ impl ErasureSetHealer {
let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
let execution = self.replacement_execution.clone();
let replacement_object = execution.is_some();
let work_parent = execution.as_ref().and(self.work_parent.as_ref()).cloned();
let failure_identity = execution
.as_ref()
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
@@ -1762,22 +1751,10 @@ impl ErasureSetHealer {
let result = if cancel_token.is_cancelled() {
(0, Err(Error::TaskCancelled))
} else {
let result = if replacement_object {
storage
.heal_object_with_task_cancel(
&bucket_name,
&object_name,
version_id.as_deref(),
&heal_opts,
&cancel_token,
)
.await
} else {
storage
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
.await
};
match result {
match storage
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
.await
{
Ok((result, None)) if target_outcomes_complete(&result, &target_endpoints) => {
let object_size = result_object_size_u64(&result);
if !replacement_commit_evidence_required {
@@ -1869,11 +1846,7 @@ impl ErasureSetHealer {
};
page_tasks.push(async move {
if let (Some(execution), Some((key, object, version))) = (execution, failure_identity) {
let result = match work_parent {
Some(parent) => execution.run_owned(&parent, work).await,
None => execution.run(work).await,
};
match result {
match execution.run(work).await {
Ok(result) => result,
Err(error) => (key, object, version, (0, Err(error)), None),
}
@@ -2785,8 +2758,6 @@ mod resume_loop_tests {
.unwrap_or(Some(HealObjectDisposition::Repaired));
let receipt = if error.is_none() {
disposition.map(|disposition| HealObjectReceipt {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: bucket.to_owned(),
@@ -145,11 +145,9 @@ pub(super) fn skipped(identity: Option<HealObjectIdentity>, reason: &str) -> Opt
identity.map(|identity| {
(
CanonicalObjectOutcome {
deep_check_observed: false,
identity,
disposition: HealObjectDisposition::Unknown,
detail: Some(reason.to_owned()),
quorum_selected_data_dir: None,
},
0,
)
@@ -251,11 +249,9 @@ pub(super) async fn heal_object(
(size, legacy),
Some((
CanonicalObjectOutcome {
deep_check_observed: false,
identity,
disposition,
detail,
quorum_selected_data_dir: None,
},
failures,
)),
@@ -420,7 +420,6 @@ async fn real_storage_grace_round_trip() {
&HealOpts {
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
recreate: true,
remove: true,
pool: Some(0),
set: Some(0),
..Default::default()
+38 -291
View File
@@ -13,20 +13,16 @@
// limitations under the License.
use crate::heal::{
outcome::{HealExecutionOutcome, HealTaskOutcome},
outcome::HealTaskOutcome,
progress::{HealProgress, HealStatistics},
resume::{
ReplacementPhase, ReplacementRecoveryBlock, ReplacementRecoveryCandidate, ResumeGc, ResumeManager, ResumeState,
ResumeUtils, merge_replacement_recovery_candidate, replacement_retry_is_exhausted_active,
},
storage::{HealStorageAPI, run_local_metadata},
storage::HealStorageAPI,
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
};
use crate::{Error, Result};
use futures::{
FutureExt,
future::{BoxFuture, Shared},
};
use metrics::{counter, gauge, histogram};
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_concurrency::workload::{ForegroundPressure, foreground_pressure};
@@ -36,10 +32,8 @@ use rustfs_heal_contracts::heal_channel::{
HealAdmissionDropReason, HealAdmissionReceipt, HealAdmissionResult, HealRequestSource,
};
use rustfs_madmin::heal_commands::HealResultItem;
use std::future::Future;
#[cfg(test)]
use std::sync::LazyLock;
use std::sync::atomic::{AtomicBool, Ordering};
use std::{
collections::{BinaryHeap, HashMap, HashSet},
sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard},
@@ -50,206 +44,11 @@ use tokio::{
time::{interval, sleep},
};
use tokio_util::sync::CancellationToken;
use tokio_util::task::{TaskTracker, task_tracker::TaskTrackerToken};
use tracing::{debug, error, info, warn};
use uuid::Uuid;
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
// Four long-lived loops have fixed receipts; finished dynamic jobs leave the
// tracker immediately. No manager/cache lock is needed by a drain waiter.
type HealWorkerJoin = Shared<BoxFuture<'static, std::result::Result<(), Arc<tokio::task::JoinError>>>>;
struct HealWorkerRegistration {
open: bool,
stopping: usize,
fixed: [Option<HealWorkerJoin>; 4],
}
#[derive(Clone)]
struct HealWorkers {
registration: Arc<StdMutex<HealWorkerRegistration>>,
tasks: TaskTracker,
startup: TaskTracker,
failed: Arc<AtomicBool>,
}
#[derive(Clone)]
pub(super) struct HealWork {
owner: HealWorkers,
_token: TaskTrackerToken,
}
// Each in-flight stop owns its own intent. Dropping one waiter cannot reopen
// startup while another stop still owns the handoff boundary.
struct HealStopIntent(HealWorkers);
impl Drop for HealStopIntent {
fn drop(&mut self) {
if let Ok(mut registration) = self.0.registration.lock() {
registration.stopping -= 1;
if registration.stopping == 0 && registration.open {
self.0.startup.reopen();
}
}
}
}
struct HealWorkGuard {
work: HealWork,
completed: bool,
}
impl Drop for HealWorkGuard {
fn drop(&mut self) {
if !self.completed {
self.work.owner.failed.store(true, Ordering::Release);
}
}
}
impl HealWork {
fn spawn<F, Fut, T>(&self, work: F) -> tokio::task::JoinHandle<T>
where
F: FnOnce(HealWork) -> Fut + Send + 'static,
Fut: Future<Output = T> + Send + 'static,
T: Send + 'static,
{
// Clone before spawn, while the live parent prevents an empty tracker.
let guard = HealWorkGuard {
work: self.clone(),
completed: false,
};
tokio::spawn(async move {
let mut guard = guard;
let output = work(guard.work.clone()).await;
guard.completed = true;
output
})
}
}
impl HealWorkers {
fn new() -> Self {
Self {
registration: Arc::new(StdMutex::new(HealWorkerRegistration {
open: true,
stopping: 0,
fixed: std::array::from_fn(|_| None),
})),
tasks: TaskTracker::new(),
startup: TaskTracker::new(),
failed: Arc::new(AtomicBool::new(false)),
}
}
fn enter_start(&self) -> Result<(HealWorkGuard, TaskTrackerToken)> {
let registration = self
.registration
.lock()
.map_err(|_| Error::other("heal worker registration poisoned"))?;
if !registration.open || registration.stopping != 0 {
return Err(Error::other("heal manager workers are stopping or terminally closed"));
}
Ok((
HealWorkGuard {
work: HealWork {
owner: self.clone(),
_token: self.tasks.token(),
},
completed: false,
},
self.startup.token(),
))
}
fn begin_stop(&self) -> Result<HealStopIntent> {
let mut registration = self
.registration
.lock()
.map_err(|_| Error::other("heal worker registration poisoned"))?;
registration.stopping += 1;
self.startup.close();
Ok(HealStopIntent(self.clone()))
}
fn spawn_fixed<F, Fut>(&self, slot: usize, work: F) -> Result<()>
where
F: FnOnce(HealWork) -> Fut + Send + 'static,
Fut: Future<Output = ()> + Send + 'static,
{
let mut registration = self
.registration
.lock()
.map_err(|_| Error::other("heal worker registration poisoned"))?;
if !registration.open || registration.fixed[slot].is_some() {
return Err(Error::other("heal manager loop is closed or already registered"));
}
let parent = HealWork {
owner: self.clone(),
_token: self.tasks.token(),
};
let task = parent.spawn(work);
registration.fixed[slot] = Some(async move { task.await.map_err(Arc::new) }.boxed().shared());
Ok(())
}
fn close(&self) -> Result<()> {
let mut registration = self
.registration
.lock()
.map_err(|_| Error::other("heal worker registration poisoned"))?;
registration.open = false;
self.tasks.close();
Ok(())
}
async fn wait(&self) -> Result<()> {
let fixed = {
let registration = self
.registration
.lock()
.map_err(|_| Error::other("heal worker registration poisoned"))?;
if registration.open {
return Err(Error::other("heal worker admission is not closed"));
}
registration.fixed.clone()
};
let mut join_failed = false;
for task in fixed.into_iter().flatten() {
join_failed |= task.await.is_err();
}
self.tasks.wait().await;
if join_failed || self.failed.load(Ordering::Acquire) {
return Err(Error::other("heal owned worker panicked or was unexpectedly cancelled"));
}
Ok(())
}
}
pub(super) fn spawn_heal_work<F, Fut, T>(parent: Option<&HealWork>, work: F) -> tokio::task::JoinHandle<T>
where
F: FnOnce(Option<HealWork>) -> Fut + Send + 'static,
Fut: Future<Output = T> + Send + 'static,
T: Send + 'static,
{
match parent {
Some(parent) => parent.spawn(move |child| work(Some(child))),
None => tokio::spawn(work(None)),
}
}
async fn inspect_resume_disk(
gc: &mut ResumeGc,
disk: &super::DiskStore,
admission: Option<&Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
cancellation: &CancellationToken,
) -> Result<()> {
run_local_metadata(disk, admission, cancellation, gc.inspect_disk(disk))
.await?
.map(|_| ())
}
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
// Each cache includes alias tokens in its count and byte budget. Eviction
// removes every token sharing a snapshot; neither cache retains repair state.
@@ -527,18 +326,9 @@ async fn publish_completed_heal(
let object = outcome
.filter(|outcome| completed.heal_type.is_per_object() && !outcome.objects_truncated && outcome.objects.len() == 1)
.and_then(|outcome| outcome.objects.front())
.map(|object| {
let quorum_selected_data_dir = object.quorum_selected_data_dir.filter(|generation| !generation.is_nil());
crate::heal::outcome::HealObjectReceipt {
identity: object.identity.clone(),
disposition: object.disposition.clone(),
quorum_selected_data_dir,
deep_check_observed: object.deep_check_observed
&& quorum_selected_data_dir.is_some()
&& !completed.options.dry_run
&& matches!(completed.status, HealTaskStatus::Completed)
&& outcome.is_some_and(|outcome| outcome.execution == HealExecutionOutcome::Completed),
}
.map(|object| crate::heal::outcome::HealObjectReceipt {
identity: object.identity.clone(),
disposition: object.disposition.clone(),
});
rustfs_heal_contracts::completion::publish_heal_completion(rustfs_heal_contracts::completion::HealCompletion {
task_id: task_id.to_owned(),
@@ -1224,7 +1014,6 @@ pub struct HealState {
/// Heal manager
pub struct HealManager {
workers: HealWorkers,
/// Heal config
config: Arc<RwLock<HealConfig>>,
/// Heal state
@@ -1399,22 +1188,15 @@ impl HealManager {
)
}
fn schedule_mainline_throttle_recheck(
notify: Arc<Notify>,
delay: Duration,
parent: Option<&HealWork>,
cancel: CancellationToken,
) {
fn schedule_mainline_throttle_recheck(notify: Arc<Notify>, delay: Duration) {
if delay.is_zero() {
notify.notify_one();
return;
}
spawn_heal_work(parent, move |_parent| async move {
tokio::select! {
_ = cancel.cancelled() => {},
_ = sleep(delay) => notify.notify_one(),
}
tokio::spawn(async move {
sleep(delay).await;
notify.notify_one();
});
}
@@ -1830,10 +1612,9 @@ impl HealManager {
/// Start the bounded resume-state inspector. Destructive GC remains
/// disabled until the durable owner/CAS contract from backlog#1927 is
/// available; this task therefore cannot remove an active or stale file.
async fn start_resume_gc(&self) -> Result<()> {
async fn start_resume_gc(&self) {
let cancel = self.cancel_token.clone();
let admission = self.storage.background_node_admission();
self.workers.spawn_fixed(2, move |_parent| async move {
tokio::spawn(async move {
let mut gc_by_disk = HashMap::<String, ResumeGc>::new();
let mut ticker = interval(RESUME_GC_INTERVAL);
loop {
@@ -1849,7 +1630,7 @@ impl HealManager {
let gc = gc_by_disk.entry(disk_key).or_default();
tokio::select! {
_ = cancel.cancelled() => return,
result = inspect_resume_disk(gc, &disk, admission.as_ref(), &cancel) => {
result = gc.inspect_disk(&disk) => {
if let Err(error) = result {
warn!(
target: "rustfs::heal::manager",
@@ -1868,14 +1649,13 @@ impl HealManager {
}
}
}
})?;
Ok(())
});
}
async fn start_root_recovery_terminal_gc(&self) -> Result<()> {
async fn start_root_recovery_terminal_gc(&self) {
let cancel = self.cancel_token.clone();
let root_recovery = self.root_recovery.clone();
self.workers.spawn_fixed(3, move |_parent| async move {
tokio::spawn(async move {
let mut ticker = interval(RESUME_GC_INTERVAL);
loop {
tokio::select! {
@@ -1914,8 +1694,7 @@ impl HealManager {
}
}
}
})?;
Ok(())
});
}
/// Create new HealManager
@@ -1940,7 +1719,6 @@ impl HealManager {
) -> Self {
let config = config.unwrap_or_default();
Self {
workers: HealWorkers::new(),
config: Arc::new(RwLock::new(config)),
state: Arc::new(RwLock::new(HealState::default())),
active_heals: Arc::new(Mutex::new(HashMap::new())),
@@ -1987,10 +1765,8 @@ impl HealManager {
/// Start HealManager
pub async fn start(&self) -> Result<()> {
let (mut starting, _startup_token) = self.workers.enter_start()?;
let mut state = self.state.write().await;
if state.is_running {
starting.completed = true;
warn!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_MANAGER_STATE,
@@ -2043,8 +1819,8 @@ impl HealManager {
self.process_unclean_shutdown().await;
// Inspect resume artifacts in a bounded, fail-closed background task.
self.start_resume_gc().await?;
self.start_root_recovery_terminal_gc().await?;
self.start_resume_gc().await;
self.start_root_recovery_terminal_gc().await;
// start auto disk scanner to heal unformatted disks
if self.config.read().await.enable_auto_heal {
@@ -2068,31 +1844,11 @@ impl HealManager {
state = "running",
"Heal manager started"
);
starting.completed = true;
Ok(())
}
#[cfg(test)]
pub(super) fn test_work_parent(&self) -> HealWork {
assert!(self.workers.registration.lock().unwrap().open, "test root parent precedes stop");
HealWork {
owner: self.workers.clone(),
_token: self.workers.tasks.token(),
}
}
/// Durable stop handoff completed; owned workers may still be joining.
pub(crate) fn stop_handoff_completed(&self) -> bool {
self.cancel_token.is_cancelled()
}
/// Stop HealManager
pub async fn stop(&self) -> Result<()> {
let _stop_intent = self.workers.begin_stop()?;
// TaskTracker registers its notification before checking closed/empty.
// Stop admission first; wait outside admin/active/cache locks so startup
// can finish its existing recovery and register all four loop receipts.
self.workers.startup.wait().await;
let _admin_start_guard = self.admin_start_shutdown.lock().await;
info!(
target: "rustfs::heal::manager",
@@ -2109,32 +1865,29 @@ impl HealManager {
let mut active_heals = self.active_heals.lock().await;
let queue = self.heal_queue.lock().await;
let retrying = self.retrying_heals.lock().await;
if !self.cancel_token.is_cancelled() {
for task in active_heals.values() {
if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) {
if task.get_status().await == HealTaskStatus::Completed {
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
} else {
let mut request = match task.retry_request_with_remaining_timeout().await {
Ok(request) => request,
Err(Error::TaskTimeout) => {
let mut request = task.retry_request();
request.options.timeout = Some(Duration::ZERO);
request
}
Err(error) => return Err(error),
};
request.retry_attempts = task.retry_attempts;
self.root_recovery.persist(&request).await?;
}
for task in active_heals.values() {
if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) {
if task.get_status().await == HealTaskStatus::Completed {
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
} else {
let mut request = match task.retry_request_with_remaining_timeout().await {
Ok(request) => request,
Err(Error::TaskTimeout) => {
let mut request = task.retry_request();
request.options.timeout = Some(Duration::ZERO);
request
}
Err(error) => return Err(error),
};
request.retry_attempts = task.retry_attempts;
self.root_recovery.persist(&request).await?;
}
}
for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) {
self.root_recovery.persist(request).await?;
}
self.workers.close()?;
self.cancel_token.cancel();
}
for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) {
self.root_recovery.persist(request).await?;
}
self.cancel_token.cancel();
drop(retrying);
drop(queue);
@@ -2178,12 +1931,6 @@ impl HealManager {
}
crate::set_heal_queue_length(0);
// Completion paths reacquire active/cache locks. Never wait while
// holding admin, active, queue, retry, cache or state guards.
drop(active_heals);
drop(_admin_start_guard);
self.workers.wait().await?;
// update state
let mut state = self.state.write().await;
state.is_running = false;
+5 -49
View File
@@ -36,7 +36,6 @@ impl HealManager {
let displaced_terminals = self.displaced_terminals.clone();
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
let storage = self.storage.clone();
let admission = storage.background_node_admission();
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
let blocks = self.replacement_recovery_blocks.clone();
let cancel_token = self.cancel_token.clone();
@@ -58,7 +57,7 @@ impl HealManager {
"Heal auto disk scanner started"
);
self.workers.spawn_fixed(1, move |_parent| async move {
tokio::spawn(async move {
let mut interval = interval(duration);
loop {
@@ -89,10 +88,6 @@ impl HealManager {
let mut conflicted_recovery_tasks = HashSet::<String>::new();
let mut conflicted_recovery_sets = HashSet::<String>::new();
let mut deferred_recovery_sets = HashSet::<String>::new();
// This tick lacks local evidence for these sets/endpoints;
// later inventory must not turn the deferral into absence.
let mut metadata_deferred_sets = HashSet::<String>::new();
let mut metadata_deferred_endpoints = HashSet::<String>::new();
let mut deferred_replacement_endpoints = HashSet::<String>::new();
let local_disks = {
let local_disk_map = local_disk_map_read().await;
@@ -138,32 +133,8 @@ impl HealManager {
continue;
}
// Preserve backend format results separately from admission deferral.
let probe = run_local_metadata(
disk, admission.as_ref(), &cancel_token, disk.get_disk_id(),
).await;
// Native capability projection can defer after the outer plan was acquired.
let resource_deferred = admission.is_some()
&& matches!(&probe, Ok(Err(DiskError::Io(error))) if error.kind() == std::io::ErrorKind::WouldBlock);
let disk_id = match probe {
Ok(result) if !resource_deferred => result,
_ => {
if cancel_token.is_cancelled() {
return;
}
metadata_deferred_endpoints.insert(endpoint.to_string());
if let Some(set) = &set_disk_id {
metadata_deferred_sets.insert(set.clone());
deferred_recovery_sets.insert(set.clone());
}
skipped_invalid_count += 1;
continue;
}
};
if admission.is_some() && cancel_token.is_cancelled() {
return;
}
match disk_id {
// detect unformatted disk via get_disk_id()
match disk.get_disk_id().await {
Err(DiskError::UnformattedDisk) => {
if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
.await
@@ -232,18 +203,8 @@ impl HealManager {
let mut recovery_disks = BTreeMap::new();
extend_replacement_recovery_disks(&mut recovery_disks, local_disks.iter().cloned());
for set_disk_id in &recovery_sets {
if metadata_deferred_sets.contains(set_disk_id) {
continue;
}
match storage.replacement_intent_disks(set_disk_id, Some(&cancel_token)).await {
match storage.replacement_intent_disks(set_disk_id).await {
Ok(disks) => extend_replacement_recovery_disks(&mut recovery_disks, disks),
Err(Error::TransientSkip { .. }) => {
if cancel_token.is_cancelled() {
return;
}
metadata_deferred_sets.insert(set_disk_id.clone());
deferred_recovery_sets.insert(set_disk_id.clone());
}
Err(error) => {
conflicted_recovery_sets.insert(set_disk_id.clone());
warn!(
@@ -269,11 +230,6 @@ impl HealManager {
let endpoint = disk.endpoint();
let disk_set_disk_id =
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
if metadata_deferred_endpoints.contains(anchor)
|| disk_set_disk_id.as_ref().is_some_and(|set| metadata_deferred_sets.contains(set))
{
continue;
}
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
Ok(task_ids) => task_ids,
Err(error) => {
@@ -839,7 +795,7 @@ impl HealManager {
}
}
}
})?;
});
Ok(())
}
}
+10 -19
View File
@@ -74,7 +74,7 @@ impl HealManager {
let workload_provider = self.workload_provider.clone();
let disposition_cache = self.disposition_cache.clone();
self.workers.spawn_fixed(0, move |parent| async move {
tokio::spawn(async move {
let mut interval = interval(config.read().await.heal_interval);
loop {
@@ -92,7 +92,7 @@ impl HealManager {
break;
}
_ = notify.notified(), if event_driven_scheduler_enable => {
Self::process_owned_heal_queue(HealQueueContext {
Self::process_heal_queue(HealQueueContext {
heal_queue: &heal_queue,
active_heals: &active_heals,
completed_heals: &completed_heals,
@@ -109,11 +109,11 @@ impl HealManager {
cancel_token: &cancel_token,
workload_provider: &workload_provider,
disposition_cache: &disposition_cache,
}, &parent)
})
.await;
}
_ = interval.tick() => {
Self::process_owned_heal_queue(HealQueueContext {
Self::process_heal_queue(HealQueueContext {
heal_queue: &heal_queue,
active_heals: &active_heals,
completed_heals: &completed_heals,
@@ -130,28 +130,19 @@ impl HealManager {
cancel_token: &cancel_token,
workload_provider: &workload_provider,
disposition_cache: &disposition_cache,
}, &parent)
})
.await;
}
}
}
})?;
});
Ok(())
}
/// Process heal queue
/// Processes multiple tasks per cycle when capacity allows and queue has high-priority items
#[cfg(test)]
pub(super) async fn process_heal_queue(context: HealQueueContext<'_>) {
Self::process_heal_queue_inner(context, None).await;
}
async fn process_owned_heal_queue(context: HealQueueContext<'_>, parent: &HealWork) {
Self::process_heal_queue_inner(context, Some(parent)).await;
}
async fn process_heal_queue_inner(context: HealQueueContext<'_>, parent: Option<&HealWork>) {
let HealQueueContext {
heal_queue,
active_heals,
@@ -263,7 +254,7 @@ impl HealManager {
let config_for_spawn = config.clone();
// start heal task
spawn_heal_work(parent, move |parent| async move {
tokio::spawn(async move {
debug!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
@@ -276,7 +267,7 @@ impl HealManager {
state = "task_started",
"Heal scheduler task started"
);
let result = task.execute_owned(parent.clone()).await;
let result = task.execute().await;
let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await;
record_task_disposition(&disposition_cache_clone, task.as_ref(), &result, retry_request.is_some()).await;
match &result {
@@ -515,7 +506,7 @@ impl HealManager {
let retry_notify = notify_clone.clone();
let retry_manager_cancel_token = manager_cancel_token.clone();
let retry_config = config_for_spawn.clone();
spawn_heal_work(parent.as_ref(), move |_parent| async move {
tokio::spawn(async move {
let mut grace_admission_backoff = Duration::ZERO;
loop {
let now = SystemTime::now()
@@ -744,7 +735,7 @@ impl HealManager {
if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure {
Self::record_mainline_throttle_delay(pressure, &config);
Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep, parent, cancel_token.clone());
Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep);
}
// Log queue status if items remain
-28
View File
@@ -28,7 +28,6 @@ use tempfile::TempDir;
pub(super) mod admin_overlap;
mod admission_timing;
mod owned_workers;
mod root_recovery;
use uuid::Uuid;
mod auto_scan;
@@ -1387,8 +1386,6 @@ fn mrf_verified_repair_event_requires_positive_exact_identity() {
durable_anchor: None,
};
let matching = HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".to_string(),
@@ -1540,8 +1537,6 @@ fn unverified_legacy_fixture(bucket: &str) -> (MrfRepairNoticeTarget, crate::hea
durable_anchor: Some(anchor.clone()),
};
let outcome = HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: bucket.to_string(),
@@ -1736,8 +1731,6 @@ fn completed_mrf_notice_publishes_only_verified_positive_events() {
outcome.execution = HealExecutionOutcome::Completed;
outcome.coverage = crate::heal::outcome::HealTraversalCoverage::Complete;
outcome.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: bucket.to_string(),
@@ -1751,8 +1744,6 @@ fn completed_mrf_notice_publishes_only_verified_positive_events() {
detail: None,
});
outcome.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: bucket.to_string(),
@@ -3122,8 +3113,6 @@ async fn local_automatic_query_preserves_source_path_token_and_state() {
},
disposition: HealObjectDisposition::VerifiedHealthy,
detail: None,
deep_check_observed: false,
quorum_selected_data_dir: None,
});
snapshot.outcome = Some(Arc::new(outcome));
snapshot.seqed_items = vec![(
@@ -6619,7 +6608,6 @@ fn admin_bucket_discovery_retries_keep_the_existing_budget() {
async fn check_node_admission_task_retry(source: HealRequestSource) {
use crate::heal::outcome::HealFailureClass;
use crate::heal::storage::ECStoreHealStorage;
use crate::heal::storage_api::owner::{PhysicalDiskSample, PhysicalDiskSampler};
use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
use rustfs_heal_contracts::heal_channel::HealScanMode;
use std::num::NonZeroU16;
@@ -6632,18 +6620,6 @@ async fn check_node_admission_task_retry(source: HealRequestSource) {
env.make_bucket("admission-retry", false).await;
env.put_object_bytes("admission-retry", "existing", b"repair me".to_vec())
.await;
let mut physical = PhysicalDiskSampler::default();
physical.publish(
PhysicalDiskSample::collect(&env.ecstore).with_test_single_device(),
ResourcePressure::Unknown,
);
assert!(
env.ecstore
.physical_heal_plan("existing", &Default::default())
.expect("fixture placement")
.is_some(),
"physical discovery must be ready before testing node capacity"
);
let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one node unit"));
let storage: Arc<dyn HealStorageAPI> =
Arc::new(ECStoreHealStorage::new(env.ecstore.clone()).with_admission(Some(gate.clone())));
@@ -6684,10 +6660,6 @@ async fn check_node_admission_task_retry(source: HealRequestSource) {
completed_units: None,
}),
);
physical.publish(
PhysicalDiskSample::collect(&env.ecstore).with_test_single_device(),
ResourcePressure::Unknown,
);
let retried = HealTask::from_request(retry, storage);
retried
.execute()
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More